From defadc8367c2f34dde764c65cd19bcfe4a9987e8 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 18:45:20 +0800 Subject: [PATCH 01/24] Add ruff, ty (default), and pyright; lock pytest-asyncio mode - ruff with FastAPI-style rule set (E/W/F/B/C4/ASYNC); `make ruff` and `make ruff-fix` targets. Keeps black/isort during transition. - ty as the default typechecker (~10x faster than pyright); pyright retained as `make typecheck-pyright`. [tool.ty.rules] enables possibly-unresolved-reference, which catches real bugs surfaced in the audit (milvus_start, user_id_list). - pytest-asyncio: pin asyncio_mode = "strict" explicitly so the current default doesn't shift under us on upstream release. - load_env.py: use os.pathsep instead of hardcoded ":" in PYTHONPATH manipulation so the env loader works on Windows. --- methods/EverCore/Makefile | 47 ++++++++++++-- methods/EverCore/pyproject.toml | 64 +++++++++++++++++++ methods/EverCore/src/common_utils/load_env.py | 6 +- 3 files changed, 107 insertions(+), 10 deletions(-) diff --git a/methods/EverCore/Makefile b/methods/EverCore/Makefile index 4619c03d..e1bba591 100644 --- a/methods/EverCore/Makefile +++ b/methods/EverCore/Makefile @@ -1,14 +1,18 @@ -.PHONY: dev-setup setup-hooks lint test clean help +.PHONY: dev-setup setup-hooks lint ruff ruff-fix typecheck typecheck-pyright test clean help # Default target help: @echo "Available targets:" - @echo " dev-setup - Full dev environment setup (sync deps + install hooks)" - @echo " setup-hooks - Install pre-commit hooks only" - @echo " lint - Run linters" - @echo " test - Run tests" - @echo " clean - Clean up generated files" - @echo " help - Show this help message" + @echo " dev-setup - Full dev environment setup (sync deps + install hooks)" + @echo " setup-hooks - Install pre-commit hooks only" + @echo " lint - Run linters (black + i18n)" + @echo " ruff - Run ruff check + format check (read-only)" + @echo " ruff-fix - Run ruff with --fix and apply formatter" + @echo " typecheck - Run ty type checker (default, fast)" + @echo " typecheck-pyright - Run pyright type checker (fallback)" + @echo " test - Run tests" + @echo " clean - Clean up generated files" + @echo " help - Show this help message" # Full development environment setup dev-setup: @@ -34,6 +38,35 @@ lint: @echo "Running i18n check..." PYTHONPATH=src python -m devops_scripts.i18n.i18n_tool check +# Run ruff (lint + format check). Read-only; CI-friendly. +ruff: + @echo "Running ruff check..." + uv run ruff check src/ + @echo "Running ruff format --check..." + uv run ruff format --check src/ + +# Apply ruff auto-fixes and formatting. +ruff-fix: + @echo "Running ruff check --fix..." + uv run ruff check --fix src/ + @echo "Running ruff format..." + uv run ruff format src/ + +# Run ty type checker (default). +# Configured in pyproject.toml under [tool.ty]. ~10x faster than pyright. +# Note: ty is pre-1.0 — pin a floor in pyproject.toml and accept that diagnostic +# wording may shift between minor releases. +typecheck: + @echo "Running ty..." + uv run ty check + +# Run pyright type checker (fallback). +# pyrightconfig.json controls scope and strictness; current mode is "off" +# (syntax/import resolution only). Tighten there to surface more checks. +typecheck-pyright: + @echo "Running pyright..." + uv run pyright + # Run tests test: PYTHONPATH=src pytest tests/ diff --git a/methods/EverCore/pyproject.toml b/methods/EverCore/pyproject.toml index 02a2912b..8862493d 100644 --- a/methods/EverCore/pyproject.toml +++ b/methods/EverCore/pyproject.toml @@ -111,12 +111,69 @@ skip-magic-trailing-comma = true profile = "black" line_length = 88 +# Ruff is introduced alongside black/isort during a transition window. +# `make ruff` runs the new lints; existing `make lint` keeps using black/isort +# so contributors' habits and CI don't break. Migrate fully in a follow-up. +[tool.ruff] +line-length = 88 +target-version = "py312" +extend-exclude = ["migrations", "__pycache__", "**/__pycache__"] + +[tool.ruff.lint] +# Conservative initial rule set. Hold back I and UP for follow-up PRs: +# - I (isort): isort already runs in `make lint`, avoid double-fixing for now +# - UP (pyupgrade): ~1500 PEP 585 / typing-modernization violations in baseline; +# deserves a dedicated migration PR with `ruff check --fix` +# Hold back G (logging f-strings) and PERF until the baseline drops. +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes (undefined names, unused/duplicate imports) + "B", # flake8-bugbear (common bugs) + "C4", # flake8-comprehensions + "ASYNC", # flake8-async (blocking calls in async, missing awaits) +] +ignore = [ + "E501", # line too long — formatter's job + "B008", # function calls in argument defaults (FastAPI uses Depends()) + "B904", # raise from inside except (large existing baseline) +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] # allow re-exports +"tests/**/*.py" = ["E741", "F811"] # ambiguous names / fixture redefinition +"unit_test/**/*.py" = ["E741", "F811"] + +[tool.ruff.format] +# Match the existing black config: preserve original quote style and don't +# magic-split-on-trailing-comma. +quote-style = "preserve" +skip-magic-trailing-comma = true + +# ty (Astral's type checker). Runs by default via `make typecheck`. +# Pre-1.0; revisit when ty hits a stable release. +[tool.ty.environment] +python-version = "3.12" +root = ["src"] + +[tool.ty.src] +include = ["src"] + +[tool.ty.rules] +# Catches "variable assigned inside try, referenced inside except" patterns +# (e.g., the milvus_start / user_id_list bugs surfaced in the audit). +possibly-unresolved-reference = "error" + [tool.pytest.ini_options] testpaths = ["tests", "unit_test"] python_files = ["test_*.py", "*_test.py"] python_classes = ["Test*"] python_functions = ["test_*"] addopts = "-v --tb=short" +# Pin pytest-asyncio mode explicitly so existing tests keep their current behavior +# (requires @pytest.mark.asyncio on async tests). Avoids silent behavior changes +# if pytest-asyncio ships a new default in a future release. +asyncio_mode = "strict" [dependency-groups] # Development dependencies - for local development @@ -132,6 +189,13 @@ dev = [ "pyinstrument", "py-spy", "pre-commit>=4.3.0", + "pyright>=1.1.400", + "ruff>=0.15.0", + # ty is Astral's type checker (Rust, ~10x faster than pyright). + # Still 0.0.x — API and diagnostic wording may change between releases. + # We pin a floor and use it as the default for `make typecheck`; + # pyright stays available via `make typecheck-pyright` as a fallback. + "ty>=0.0.39", ] cpu = [ diff --git a/methods/EverCore/src/common_utils/load_env.py b/methods/EverCore/src/common_utils/load_env.py index 6561a509..c99288e8 100644 --- a/methods/EverCore/src/common_utils/load_env.py +++ b/methods/EverCore/src/common_utils/load_env.py @@ -95,8 +95,8 @@ def sync_pythonpath_with_syspath(): import os from pathlib import Path - # Get current PYTHONPATH - pythonpath = os.environ.get("PYTHONPATH", "").split(":") + # Get current PYTHONPATH (os.pathsep is ":" on POSIX and ";" on Windows) + pythonpath = os.environ.get("PYTHONPATH", "").split(os.pathsep) pythonpath = [p for p in pythonpath if p] # Remove empty strings # Path patterns to exclude @@ -138,7 +138,7 @@ def sync_pythonpath_with_syspath(): # Append new paths to existing PYTHONPATH all_paths = pythonpath + new_paths # Update environment variable - os.environ["PYTHONPATH"] = ":".join(all_paths) + os.environ["PYTHONPATH"] = os.pathsep.join(all_paths) logger.debug("Updated PYTHONPATH: %s", os.environ["PYTHONPATH"]) From fab8e010dae26cc353b98cef1affe886579c4a83 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 18:45:29 +0800 Subject: [PATCH 02/24] Fix possibly-unbound vars and duplicate imports surfaced by ty/ruff - memory_manager.get_vector_search_results: initialize milvus_start before the try so the except branch can always compute the stage duration. Previously a failure before line 729 would raise UnboundLocalError, masking the real exception. - mem_memorize._trigger_profile_extraction: initialize user_id_list before the try so the recovery branch can iterate participants safely on early failure. - mem_memorize: `request == None` -> `request is None` (E711). - mem_memorize / memory_manager / mem_db_operations: remove duplicate imports of traceback, dataclass, RawDataType, MemorizeConfig, DEFAULT_MEMORIZE_CONFIG, Any, MemoryType (F811). - retrieval_utils: bare `except:` -> `except Exception:` to stop swallowing KeyboardInterrupt / SystemExit (E722). - i18n_tool: declare OpenAIProvider under TYPE_CHECKING so the forward-reference annotations resolve without forcing memory_layer to be importable at module load. --- .../src/agentic_layer/memory_manager.py | 11 ++++++++--- .../src/agentic_layer/retrieval_utils.py | 2 +- .../src/biz_layer/mem_db_operations.py | 2 +- .../EverCore/src/biz_layer/mem_memorize.py | 19 +++++++------------ .../src/devops_scripts/i18n/i18n_tool.py | 8 +++++++- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index e6e93005..22d8f9c6 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional import logging import asyncio @@ -8,7 +8,6 @@ import jieba import numpy as np import time -from typing import Dict, Any from dataclasses import dataclass from api_specs import memory_types @@ -93,7 +92,7 @@ PROFILE_RECALL_THRESHOLD, PROFILE_DEFAULT_TOPK, ) -from api_specs.memory_models import MemoryType, RetrieveMethod +from api_specs.memory_models import RetrieveMethod from agentic_layer.metrics.retrieve_metrics import ( record_retrieve_stage, record_retrieve_error, @@ -620,6 +619,12 @@ async def get_vector_search_results( else 'unknown' ) + # Initialize milvus_start so the except branch can always compute a + # duration. If the exception fires before the inner re-assignment, + # the recorded value will reflect time-from-function-entry — still + # better than NameError-on-error masking the original failure. + milvus_start = time.perf_counter() + try: # Get parameters from Request logger.debug( diff --git a/methods/EverCore/src/agentic_layer/retrieval_utils.py b/methods/EverCore/src/agentic_layer/retrieval_utils.py index 713b0aac..955611d8 100644 --- a/methods/EverCore/src/agentic_layer/retrieval_utils.py +++ b/methods/EverCore/src/agentic_layer/retrieval_utils.py @@ -237,7 +237,7 @@ async def lightweight_retrieval( if doc_norm > 0: sim = np.dot(query_vec, doc_vec) / (query_norm * doc_norm) scores.append((mem, float(sim))) - except: + except Exception: continue emb_results = sorted(scores, key=lambda x: x[1], reverse=True)[:emb_top_n] diff --git a/methods/EverCore/src/biz_layer/mem_db_operations.py b/methods/EverCore/src/biz_layer/mem_db_operations.py index c5a698a0..4ed48c14 100644 --- a/methods/EverCore/src/biz_layer/mem_db_operations.py +++ b/methods/EverCore/src/biz_layer/mem_db_operations.py @@ -38,7 +38,7 @@ from infra_layer.adapters.out.persistence.document.memory.atomic_fact_record import ( AtomicFactRecord, ) -from api_specs.memory_types import RawDataType, AgentCase +from api_specs.memory_types import AgentCase from infra_layer.adapters.out.persistence.document.memory.agent_case import ( AgentCaseRecord, ) diff --git a/methods/EverCore/src/biz_layer/mem_memorize.py b/methods/EverCore/src/biz_layer/mem_memorize.py index be1adcbc..ab1578f0 100644 --- a/methods/EverCore/src/biz_layer/mem_memorize.py +++ b/methods/EverCore/src/biz_layer/mem_memorize.py @@ -23,7 +23,7 @@ AgentCase, ) from api_specs.memory_types import AtomicFact, get_text_from_content_items -from biz_layer.memorize_config import DEFAULT_MEMORIZE_CONFIG +from biz_layer.memorize_config import MemorizeConfig, DEFAULT_MEMORIZE_CONFIG from core.di import get_bean_by_type from infra_layer.adapters.out.persistence.repository.episodic_memory_raw_repository import ( EpisodicMemoryRawRepository, @@ -44,9 +44,7 @@ from infra_layer.adapters.out.persistence.repository.conversation_data_raw_repository import ( ConversationDataRepository, ) -from api_specs.memory_types import RawDataType from typing import List, Dict, Optional, Any -from dataclasses import dataclass import uuid from datetime import datetime, timedelta import os @@ -54,7 +52,6 @@ from collections import defaultdict from common_utils.datetime_utils import get_now_with_timezone, to_iso_format from memory_layer.memcell_extractor.base_memcell_extractor import StatusResult -import traceback from core.observation.logger import get_logger from infra_layer.adapters.out.search.elasticsearch.converter.episodic_memory_converter import ( @@ -86,12 +83,6 @@ class MemoryDocPayload: doc: Any -from biz_layer.memorize_config import ( - MemorizeConfig, - DEFAULT_MEMORIZE_CONFIG, -) - - def _is_agent_case_quality_sufficient( agent_case: AgentCase, config: MemorizeConfig ) -> bool: @@ -394,6 +385,10 @@ async def _trigger_profile_extraction( scene: Conversation scene config: Memory extraction configuration """ + # Initialize so the except branch can iterate even if failure happens + # before the in-try assignment at the participant-aggregation step. + user_id_list: List[str] = [] + try: from memory_layer.profile_manager import ProfileManager, ProfileManagerConfig from infra_layer.adapters.out.persistence.repository.user_profile_raw_repository import ( @@ -1709,8 +1704,8 @@ async def memorize(request: MemorizeRequest) -> int: ): with timed("validate_request"): request = await preprocess_conv_request(request, current_time) - if request == None: - logger.warning(f"[mem_memorize] preprocess_conv_request returned None") + if request is None: + logger.warning("[mem_memorize] preprocess_conv_request returned None") return 0 # Fetch llm_custom_setting from global config (inherits automatically) diff --git a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py index 89c29c6d..ee382279 100644 --- a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py +++ b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py @@ -46,8 +46,14 @@ import subprocess from fnmatch import fnmatch from pathlib import Path -from typing import Optional +from typing import Optional, TYPE_CHECKING from dataclasses import dataclass + +if TYPE_CHECKING: + # OpenAIProvider is only used in type annotations; the runtime import + # lives inside _load_env_and_get_llm_provider() so the module doesn't + # depend on memory_layer being importable just for type-checking. + from memory_layer.llm import OpenAIProvider from enum import Enum # ============================================================================== From 7c7082056c10e46b1530259ed4a9d83aba5c7a70 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 18:45:36 +0800 Subject: [PATCH 03/24] Remove dead GET-variant SearchMemoriesResponse Two classes named SearchMemoriesResponse were defined in api_specs/dtos/memory.py: a GET-endpoint variant at line 774 (BaseApiResponse[RetrieveMemResponse]) and a POST-endpoint variant at line 1697 (BaseApiResponse[SearchMemoriesResponseData]). The second definition silently shadowed the first, making the GET variant dead code. The POST variant is the one wired to memory_search_controller (it instantiates `SearchMemoriesResponse(data=response_data)` where response_data is SearchMemoriesResponseData), so keep that one and remove the shadowed GET variant + its example schema. --- methods/EverCore/src/api_specs/dtos/memory.py | 58 ------------------- 1 file changed, 58 deletions(-) diff --git a/methods/EverCore/src/api_specs/dtos/memory.py b/methods/EverCore/src/api_specs/dtos/memory.py index 2ee9d04f..c1bafbc9 100644 --- a/methods/EverCore/src/api_specs/dtos/memory.py +++ b/methods/EverCore/src/api_specs/dtos/memory.py @@ -771,64 +771,6 @@ class RetrieveMemResponse(BaseModel): model_config = {"arbitrary_types_allowed": True} -class SearchMemoriesResponse(BaseApiResponse[RetrieveMemResponse]): - """Memory search API response - - Response for GET /api/v1/memories/search endpoint. - """ - - data: RetrieveMemResponse = Field(description="Memory search result") - - model_config = { - "json_schema_extra": { - "example": { - "status": "ok", - "message": "Memory search successful", - "result": { - "profiles": [ - { - "item_type": "explicit_info", - "category": "Dietary Preferences", - "description": "Prefers light flavors, favoring vegetables and seafood", - "score": 0.89, - }, - { - "item_type": "implicit_trait", - "trait_name": "Health Conscious", - "description": "Prioritizes dietary health, preferring low oil and low salt", - "score": 0.75, - }, - ], - "memories": [ - { - "memory_type": "episodic_memory", - "user_id": "user_123", - "timestamp": "2024-01-15T10:30:00", - "summary": "User mentioned controlling their diet recently, eating only two meals a day, with dinner mainly being salad", - "group_id": "group_456", - } - ], - "scores": [0.82], - "original_data": [], - "total_count": 3, - "has_more": False, - "query_metadata": { - "source": "hybrid_search", - "user_id": "user_123", - "memory_type": "retrieve", - }, - "metadata": { - "profile_count": 2, - "episodic_count": 1, - "latency_ms": 156, - }, - "pending_messages": [], - }, - } - } - } - - # ============================================================================= # Delete DTOs (DELETE /api/v1/memories) # ============================================================================= From bf48484adc59c27cde99ad4c401042be749af171 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 18:51:29 +0800 Subject: [PATCH 04/24] Fix possibly-unbound vars surfaced by ty (second pass) Same "assigned inside try, used in except / after-try" pattern as the earlier milvus_start / user_id_list fixes: - mem_memorize._trigger_profile_extraction: initialize profile_repo to None before the try, and skip the last_updated_ts advance when the DI lookup never ran (failure during lazy imports). - memory_layer.memory_manager: initialize display_name to None before the if/else so the else-branch loop falling through (no matching sender_id) doesn't leave it unbound at the downstream call site. - milvus_collection_base: pull `new_coll = Collection(...)` outside the index-creation try so an index-warning path can't shadow the collection-creation success with a NameError on `return new_coll`. - hmac_signature_middleware: bind timestamp_header / nonce_header / signature_header to None before the try, so the except branch's error logging works even if request.headers itself raises. - redis_length_cache_manager, redis_windows_cache_manager: bind `message = None` at the top of each loop iteration so the except branch can format its warning without a NameError when indexing raises before the assignment. Incidentally includes ruff F541 cleanup (f-strings without placeholders -> plain strings) in mem_memorize.py from the same auto-fix pass. --- methods/EverCore/src/biz_layer/mem_memorize.py | 16 +++++++++++----- .../redis_length_cache_manager.py | 3 +++ .../redis_windows_cache_manager.py | 3 +++ .../core/middleware/hmac_signature_middleware.py | 6 ++++++ .../core/oxm/milvus/milvus_collection_base.py | 6 +++++- .../EverCore/src/memory_layer/memory_manager.py | 3 +++ 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/methods/EverCore/src/biz_layer/mem_memorize.py b/methods/EverCore/src/biz_layer/mem_memorize.py index ab1578f0..2376f6f7 100644 --- a/methods/EverCore/src/biz_layer/mem_memorize.py +++ b/methods/EverCore/src/biz_layer/mem_memorize.py @@ -132,7 +132,7 @@ async def _trigger_clustering( ) from core.di import get_bean_by_type - logger.info(f"[Clustering] Retrieving MemSceneRawRepository...") + logger.info("[Clustering] Retrieving MemSceneRawRepository...") # Get MongoDB storage cluster_storage = get_bean_by_type(MemSceneRawRepository) logger.info( @@ -235,7 +235,7 @@ async def _trigger_clustering( ) await cluster_storage.save_mem_scene(group_id, mem_scene_state.to_dict()) - logger.info(f"[Clustering] Clustering state saved") + logger.info("[Clustering] Clustering state saved") if cluster_id: logger.debug( @@ -388,6 +388,9 @@ async def _trigger_profile_extraction( # Initialize so the except branch can iterate even if failure happens # before the in-try assignment at the participant-aggregation step. user_id_list: List[str] = [] + # profile_repo is resolved via DI inside the try; the recovery branch + # checks for None before attempting to advance last_updated_ts. + profile_repo = None try: from memory_layer.profile_manager import ProfileManager, ProfileManagerConfig @@ -526,6 +529,9 @@ async def _trigger_profile_extraction( # Advance last_updated_ts even on failure to prevent repeated re-selection # of the same clusters. The data is "skipped" — acceptable tradeoff vs. # getting stuck in a loop retrying the same failing extraction. + if profile_repo is None: + # Failure happened before DI lookup; nothing to advance. + return try: memcell_ts = memcell.timestamp.timestamp() if memcell.timestamp else 0.0 for uid in user_id_list: @@ -1444,12 +1450,12 @@ async def update_status_when_no_memcell( if status_result.should_wait: logger.info( - f"[mem_memorize] Determined as unable to decide boundary, continue waiting, no status update" + "[mem_memorize] Determined as unable to decide boundary, continue waiting, no status update" ) return else: logger.info( - f"[mem_memorize] Determined as non-boundary, continue accumulating messages, update status table" + "[mem_memorize] Determined as non-boundary, continue accumulating messages, update status table" ) # Get latest message timestamp latest_time = to_iso_format(current_time) @@ -1500,7 +1506,7 @@ async def update_status_after_memcell( ) logger.info( - f"[mem_memorize] Memory extraction completed, status table updated" + "[mem_memorize] Memory extraction completed, status table updated" ) except Exception as e: diff --git a/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py b/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py index ee11d18e..87791647 100644 --- a/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py +++ b/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py @@ -431,6 +431,9 @@ async def get_by_timestamp_range( return [] for i in range(0, len(messages), 2): + # Bind early so the except branch can log it even if the + # indexing below raises before `message` would be assigned. + message = None try: if i + 1 >= len(messages): logger.warning( diff --git a/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py b/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py index 9575b639..73a13fe4 100644 --- a/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py +++ b/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py @@ -416,6 +416,9 @@ async def get_by_timestamp_range( return [] for i in range(0, len(messages), 2): + # Bind early so the except branch can log it even if the + # indexing below raises before `message` would be assigned. + message = None try: if i + 1 >= len(messages): logger.warning( diff --git a/methods/EverCore/src/core/middleware/hmac_signature_middleware.py b/methods/EverCore/src/core/middleware/hmac_signature_middleware.py index cef0b361..4aea72ff 100644 --- a/methods/EverCore/src/core/middleware/hmac_signature_middleware.py +++ b/methods/EverCore/src/core/middleware/hmac_signature_middleware.py @@ -75,6 +75,12 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: # Set user context token token = None + # Initialize header vars so the except branch's error logging works + # even if request.headers itself raises before they're assigned. + timestamp_header = None + nonce_header = None + signature_header = None + # Step 1: Attempt HMAC signature verification and set user context try: # Get timestamp, nonce, and signature from request headers diff --git a/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py b/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py index ebb1b269..c590bd54 100644 --- a/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py +++ b/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py @@ -541,9 +541,13 @@ def create_new_collection(self) -> Collection: create_kwargs["num_partitions"] = num_partitions Collection(**create_kwargs) + # Bind the new collection handle outside the try so that an index- + # creation failure (warning-only path) doesn't prevent us from + # returning the collection that we just successfully created above. + new_coll = Collection(name=new_real_name, using=self._using) + # Create indexes for new collection try: - new_coll = Collection(name=new_real_name, using=self._using) self._create_indexes_for_collection(new_coll) new_coll.load() except Exception as e: diff --git a/methods/EverCore/src/memory_layer/memory_manager.py b/methods/EverCore/src/memory_layer/memory_manager.py index e496f5c5..beec525d 100644 --- a/methods/EverCore/src/memory_layer/memory_manager.py +++ b/methods/EverCore/src/memory_layer/memory_manager.py @@ -373,6 +373,9 @@ async def _extract_foresight( conversation_text = "\n".join(lines) # Best-effort resolve user_name from raw messages + # Default keeps display_name bound when the else-branch loop finds + # no matching sender_id, so the downstream call below doesn't raise. + display_name = None if uid is None: display_name = ",".join( From db316fe93b4940af79036f02a36751a873b6701e Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 18:51:40 +0800 Subject: [PATCH 05/24] Auto-fix whitespace and empty f-strings via ruff Mechanical cleanup from `ruff check --fix --select W291,W293,F541, C420,B010,B009`: - W293 blank-line-with-whitespace: removed trailing whitespace on otherwise-empty lines (~126 occurrences). - W291 trailing-whitespace: trimmed line-end whitespace. - F541 f-string-missing-placeholders: rewrote `f"..."` literals with no interpolation as plain strings (~22 occurrences). - C420 unnecessary-dict-comprehension-for-iterable, B010 set-attr- with-constant, B009 get-attr-with-constant: simplified a handful of unnecessary dict comprehensions and setattr/getattr-with- constant-attribute calls into direct attribute access. No behavior changes. The remaining 144 W293 / 9 W291 occurrences are inside docstrings or string literals and require --unsafe-fixes, which is intentionally not applied. --- .../src/agentic_layer/metrics/__init__.py | 10 +-- .../agentic_layer/metrics/rerank_metrics.py | 4 +- .../metrics/vectorize_metrics.py | 6 +- .../agentic_layer/profile_search_service.py | 54 ++++++------- .../EverCore/src/agentic_layer/rerank_vllm.py | 2 +- .../src/api_specs/request_converter.py | 2 +- methods/EverCore/src/app.py | 2 +- .../src/biz_layer/mem_db_operations.py | 12 +-- methods/EverCore/src/bootstrap.py | 4 +- .../EverCore/src/common_utils/text_utils.py | 4 +- .../EverCore/src/core/authorize/decorators.py | 2 +- .../core/component/kafka_consumer_factory.py | 2 +- .../llm/llm_adapter/gemini_client.py | 4 +- .../llm/tokenizer/tokenizer_factory.py | 8 +- methods/EverCore/src/core/di/utils.py | 2 +- .../interface/controller/base_controller.py | 6 +- .../src/core/lifespan/metrics_lifespan.py | 12 +-- .../core/middleware/prometheus_middleware.py | 24 +++--- .../src/core/observation/metrics/__init__.py | 8 +- .../src/core/observation/metrics/counter.py | 12 +-- .../src/core/observation/metrics/gauge.py | 76 +++++++++---------- .../src/core/observation/metrics/histogram.py | 28 +++---- .../src/core/observation/metrics/server.py | 10 +-- methods/EverCore/src/core/oxm/es/doc_base.py | 2 +- .../src/devops_scripts/i18n/i18n_tool.py | 4 +- .../memory_extractor/atomic_fact_extractor.py | 6 +- .../memory_extractor/foresight_extractor.py | 2 +- 27 files changed, 154 insertions(+), 154 deletions(-) diff --git a/methods/EverCore/src/agentic_layer/metrics/__init__.py b/methods/EverCore/src/agentic_layer/metrics/__init__.py index 11bb5d05..b419ae93 100644 --- a/methods/EverCore/src/agentic_layer/metrics/__init__.py +++ b/methods/EverCore/src/agentic_layer/metrics/__init__.py @@ -57,31 +57,31 @@ 'VECTORIZE_FALLBACK_TOTAL', 'VECTORIZE_ERRORS_TOTAL', 'VECTORIZE_TOKENS_TOTAL', - + # Rerank metrics 'RERANK_REQUESTS_TOTAL', 'RERANK_DURATION_SECONDS', 'RERANK_DOCUMENTS_TOTAL', 'RERANK_FALLBACK_TOTAL', 'RERANK_ERRORS_TOTAL', - + # Retrieve metrics 'RETRIEVE_REQUESTS_TOTAL', 'RETRIEVE_DURATION_SECONDS', 'RETRIEVE_RESULTS_COUNT', 'RETRIEVE_STAGE_DURATION_SECONDS', 'RETRIEVE_ERRORS_TOTAL', - + # Memorize metrics 'MEMORIZE_REQUESTS_TOTAL', 'MEMORIZE_DURATION_SECONDS', 'MEMORIZE_MESSAGES_TOTAL', 'MEMORIZE_ERRORS_TOTAL', - + # Boundary detection metrics 'BOUNDARY_DETECTION_TOTAL', 'MEMCELL_EXTRACTED_TOTAL', - + # Memory extraction metrics 'MEMORY_EXTRACTION_STAGE_DURATION_SECONDS', 'MEMORY_EXTRACTED_TOTAL', diff --git a/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py b/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py index a993dbaf..071724ea 100644 --- a/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py @@ -157,12 +157,12 @@ def record_rerank_request( provider=provider, status=status ).inc() - + # Duration histogram RERANK_DURATION_SECONDS.labels( provider=provider ).observe(duration_seconds) - + # Documents count histogram RERANK_DOCUMENTS_TOTAL.labels( provider=provider diff --git a/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py b/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py index 5ea2696e..200987d6 100644 --- a/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py @@ -159,20 +159,20 @@ def record_vectorize_request( operation=operation, status=status ).inc() - + # Duration histogram VECTORIZE_DURATION_SECONDS.labels( provider=provider, operation=operation ).observe(duration_seconds) - + # Batch size histogram (only for batch operations) if batch_size > 1: VECTORIZE_BATCH_SIZE.labels( provider=provider, operation=operation ).observe(batch_size) - + # Token counter (if available) if tokens > 0: VECTORIZE_TOKENS_TOTAL.labels(provider=provider).inc(tokens) diff --git a/methods/EverCore/src/agentic_layer/profile_search_service.py b/methods/EverCore/src/agentic_layer/profile_search_service.py index a86f5de6..3260489c 100644 --- a/methods/EverCore/src/agentic_layer/profile_search_service.py +++ b/methods/EverCore/src/agentic_layer/profile_search_service.py @@ -44,17 +44,17 @@ def parse_embed_text(embed_text: str, item_type: str) -> Dict[str, str]: Dict with parsed fields """ result = {} - + if not embed_text: return {"category": "", "description": ""} if item_type == "explicit_info" else {"trait_name": "", "description": ""} - + # Split by first colon parts = embed_text.split(":", 1) - + if len(parts) == 2: key = parts[0].strip() value = parts[1].strip() - + if item_type == "explicit_info": result["category"] = key result["description"] = value @@ -71,7 +71,7 @@ def parse_embed_text(embed_text: str, item_type: str) -> Dict[str, str]: else: result["trait_name"] = "" result["description"] = embed_text - + return result @@ -83,7 +83,7 @@ class ProfileSearchService: Searches user profile items in Milvus using vector similarity. No reranking step - directly returns Milvus results with score threshold. """ - + def __init__( self, milvus_repo: Optional[UserProfileMilvusRepository] = None, @@ -94,14 +94,14 @@ def __init__( milvus_repo: User profile Milvus repository (auto-injected if None) """ self._milvus_repo = milvus_repo - + @property def milvus_repo(self) -> UserProfileMilvusRepository: """Lazy load Milvus repository""" if self._milvus_repo is None: self._milvus_repo = get_bean_by_type(UserProfileMilvusRepository) return self._milvus_repo - + async def search_profiles( self, query: str, @@ -126,7 +126,7 @@ async def search_profiles( - metadata: Search metadata (latency, count, etc.) """ start_time = time.perf_counter() - + result = { "profiles": [], "metadata": { @@ -134,29 +134,29 @@ async def search_profiles( "latency_ms": 0, } } - + if not query: logger.warning("Empty query for profile search") return result - + try: # Step 1: Generate query embedding vectorize_service = get_vectorize_service() query_vector = await vectorize_service.get_embedding(query) - + if query_vector is None or len(query_vector) == 0: logger.warning("Failed to generate query embedding") return result - + # Step 2: Search Milvus (recall with threshold) # Recall more candidates, then filter by threshold recall_limit = top_k * 2 if top_k > 0 else PROFILE_DEFAULT_TOPK * 2 - + logger.info( f"🔍 Profile search params: user_id={user_id}, group_id={group_id}, " f"top_k={top_k}, recall_limit={recall_limit}, score_threshold={score_threshold}" ) - + milvus_results = await self.milvus_repo.vector_search( query_vector=query_vector, user_id=user_id, @@ -164,25 +164,25 @@ async def search_profiles( limit=recall_limit, score_threshold=score_threshold, ) - + logger.info( f"✅ Milvus returned {len(milvus_results)} results, will take top {top_k}" ) - + # Step 3: Process results - parse embed_text and format output profiles = [] for item in milvus_results[:top_k]: item_type = item.get("item_type", "") embed_text = item.get("embed_text", "") - + # Parse embed_text to get category/trait_name and description parsed = parse_embed_text(embed_text, item_type) - + profile_item = { "item_type": item_type, "score": round(item.get("score", 0.0), 4), } - + # Add parsed fields based on item_type if item_type == "explicit_info": profile_item["category"] = parsed.get("category", "") @@ -190,23 +190,23 @@ async def search_profiles( else: # implicit_trait profile_item["trait_name"] = parsed.get("trait_name", "") profile_item["description"] = parsed.get("description", "") - + profiles.append(profile_item) - + # Calculate latency latency_ms = int((time.perf_counter() - start_time) * 1000) - + result["profiles"] = profiles result["metadata"]["profile_count"] = len(profiles) result["metadata"]["latency_ms"] = latency_ms - + # Log profile scores for debugging if profiles: scores_str = ", ".join([f"{p['score']:.4f}" for p in profiles]) logger.info( f"📊 Profile scores: [{scores_str}]" ) - + logger.info( "Profile search completed: user_id=%s, group_id=%s, " "query='%s', found=%d, latency=%dms", @@ -216,9 +216,9 @@ async def search_profiles( len(profiles), latency_ms, ) - + return result - + except Exception as e: logger.error( "Profile search failed: user_id=%s, group_id=%s, error=%s", diff --git a/methods/EverCore/src/agentic_layer/rerank_vllm.py b/methods/EverCore/src/agentic_layer/rerank_vllm.py index 3cf9f75b..6545b50f 100644 --- a/methods/EverCore/src/agentic_layer/rerank_vllm.py +++ b/methods/EverCore/src/agentic_layer/rerank_vllm.py @@ -267,7 +267,7 @@ async def rerank_memories( # Parse results (OpenAI-compatible format) if "results" not in result: raise RerankError( - f"Invalid rerank response format: missing 'results' key" + "Invalid rerank response format: missing 'results' key" ) # Create score mapping diff --git a/methods/EverCore/src/api_specs/request_converter.py b/methods/EverCore/src/api_specs/request_converter.py index 1259daa9..1250e181 100644 --- a/methods/EverCore/src/api_specs/request_converter.py +++ b/methods/EverCore/src/api_specs/request_converter.py @@ -652,7 +652,7 @@ def convert_agent_add_to_memorize_request( elif role == "assistant": if sender_id and sender_id == user_id: raise ValueError( - f"sender_id conflict: role=assistant cannot use user_id as sender_id" + "sender_id conflict: role=assistant cannot use user_id as sender_id" ) if not sender_id: hash_val = hashlib.md5(f"{user_id}_assistant".encode()).hexdigest()[:12] diff --git a/methods/EverCore/src/app.py b/methods/EverCore/src/app.py index 986ccc8d..fa445369 100644 --- a/methods/EverCore/src/app.py +++ b/methods/EverCore/src/app.py @@ -124,7 +124,7 @@ def create_business_app( fastapi_app.user_middleware.append(Middleware(AppLogicMiddleware)) # Not directly interfacing with users # fastapi_app.user_middleware.append(Middleware(UserContextMiddleware)) - + # Add Prometheus HTTP metrics middleware fastapi_app.user_middleware.append(Middleware(PrometheusMiddleware)) diff --git a/methods/EverCore/src/biz_layer/mem_db_operations.py b/methods/EverCore/src/biz_layer/mem_db_operations.py index 4ed48c14..dcb9d3a8 100644 --- a/methods/EverCore/src/biz_layer/mem_db_operations.py +++ b/methods/EverCore/src/biz_layer/mem_db_operations.py @@ -418,16 +418,16 @@ async def _update_status_for_continuing_conversation( "updated_at": current_time, } - logger.debug(f"Conversation continuing, update new_msg_start_time") + logger.debug("Conversation continuing, update new_msg_start_time") result = await status_repo.upsert_by_group_id( request.group_id, update_data, session_id=request.session_id ) if result: - logger.info(f"Conversation continuation status updated successfully") + logger.info("Conversation continuation status updated successfully") return True else: - logger.warning(f"Conversation continuation status update failed") + logger.warning("Conversation continuation status update failed") return False except Exception as e: @@ -521,16 +521,16 @@ async def _update_status_after_memcell_extraction( # TODO : clear queue - logger.debug(f"Update status table after MemCell extraction") + logger.debug("Update status table after MemCell extraction") result = await status_repo.upsert_by_group_id( request.group_id, update_data, session_id=request.session_id ) if result: - logger.info(f"Status update after MemCell extraction successful") + logger.info("Status update after MemCell extraction successful") return True else: - logger.warning(f"Status update after MemCell extraction failed") + logger.warning("Status update after MemCell extraction failed") return False except Exception as e: diff --git a/methods/EverCore/src/bootstrap.py b/methods/EverCore/src/bootstrap.py index 2c0b14ba..5b21698c 100644 --- a/methods/EverCore/src/bootstrap.py +++ b/methods/EverCore/src/bootstrap.py @@ -202,7 +202,7 @@ async def async_main(): # If relative import error occurs, try running in module mode if "attempted relative import with no known parent package" in str(e): print( - f"\n⚠️ Detected relative import error, trying to run in module mode..." + "\n⚠️ Detected relative import error, trying to run in module mode..." ) try: # Get src directory path @@ -237,7 +237,7 @@ async def async_main(): print(f"\n📋 Script exited with code: {e.code}") raise # Re-raise to propagate the exit code else: - print(f"\n📋 Script execution completed successfully") + print("\n📋 Script execution completed successfully") except Exception as e: print(f"\n❌ Script execution error: {e}", file=sys.stderr) import traceback diff --git a/methods/EverCore/src/common_utils/text_utils.py b/methods/EverCore/src/common_utils/text_utils.py index 55700e4e..b9ceda9e 100644 --- a/methods/EverCore/src/common_utils/text_utils.py +++ b/methods/EverCore/src/common_utils/text_utils.py @@ -366,8 +366,8 @@ def get_text_analysis(self, text: str) -> Dict[str, Any]: tokens = self.parse_tokens(text) # Count tokens by type - type_counts = {token_type: 0 for token_type in TokenType} - type_scores = {token_type: 0.0 for token_type in TokenType} + type_counts = dict.fromkeys(TokenType, 0) + type_scores = dict.fromkeys(TokenType, 0.0) for token in tokens: type_counts[token.type] += 1 diff --git a/methods/EverCore/src/core/authorize/decorators.py b/methods/EverCore/src/core/authorize/decorators.py index 0c9189b3..e81f9144 100644 --- a/methods/EverCore/src/core/authorize/decorators.py +++ b/methods/EverCore/src/core/authorize/decorators.py @@ -38,7 +38,7 @@ def decorator(func: Callable) -> Callable: ) # Store authorization info on the function - setattr(func, '__authorization_context__', auth_context) + func.__authorization_context__ = auth_context @functools.wraps(func) async def async_wrapper(*args, **kwargs): diff --git a/methods/EverCore/src/core/component/kafka_consumer_factory.py b/methods/EverCore/src/core/component/kafka_consumer_factory.py index 9da07b80..39c0c5f1 100644 --- a/methods/EverCore/src/core/component/kafka_consumer_factory.py +++ b/methods/EverCore/src/core/component/kafka_consumer_factory.py @@ -419,7 +419,7 @@ async def seek_to_datetime( ) from e # Build timestamp map for each partition - timestamp_map = {partition: target_timestamp_ms for partition in partitions} + timestamp_map = dict.fromkeys(partitions, target_timestamp_ms) # Use offsets_for_times to get the offset at the corresponding timestamp offset_map = await consumer.offsets_for_times(timestamp_map) diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py index f0de5d5d..ef764564 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py @@ -207,8 +207,8 @@ def _convert_messages_to_gemini_format( try: # Check if it has role and content attributes if hasattr(msg, 'role') and hasattr(msg, 'content'): - role = getattr(msg, 'role') - content = getattr(msg, 'content') + role = msg.role + content = msg.content gemini_role = self._map_role_to_gemini(role) contents.append( ContentDict(role=gemini_role, parts=[{"text": str(content)}]) diff --git a/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py b/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py index 1f81231d..ef1698e8 100644 --- a/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py +++ b/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py @@ -50,12 +50,12 @@ def get_tokenizer_from_tiktoken(self, encoding_name: str) -> tiktoken.Encoding: >>> tokens = tokenizer.encode("Hello, world!") """ cache_key = f"tiktoken:{encoding_name}" - + if cache_key not in self._tokenizers: logger.debug("Loading tiktoken encoding: %s", encoding_name) self._tokenizers[cache_key] = tiktoken.get_encoding(encoding_name) logger.debug("Tiktoken encoding '%s' loaded and cached", encoding_name) - + return self._tokenizers[cache_key] def load_default_encodings(self) -> None: @@ -68,14 +68,14 @@ def load_default_encodings(self) -> None: The encodings loaded are defined in DEFAULT_TIKTOKEN_ENCODINGS. """ logger.info("Preloading %d tiktoken encodings...", len(DEFAULT_TIKTOKEN_ENCODINGS)) - + for encoding_name in DEFAULT_TIKTOKEN_ENCODINGS: try: self.get_tokenizer_from_tiktoken(encoding_name) logger.info("Successfully preloaded tiktoken encoding: %s", encoding_name) except (ValueError, KeyError, RuntimeError) as e: logger.error("Failed to preload tiktoken encoding '%s': %s", encoding_name, e) - + logger.info("Tiktoken encodings preload completed") def get_cached_tokenizer_count(self) -> int: diff --git a/methods/EverCore/src/core/di/utils.py b/methods/EverCore/src/core/di/utils.py index 9a1d1c41..74a29532 100644 --- a/methods/EverCore/src/core/di/utils.py +++ b/methods/EverCore/src/core/di/utils.py @@ -389,7 +389,7 @@ def print_container_info(): info, ) # Convenient usage, suitable for occasional calls - info(f"\n📦 Dependency injection container information:") + info("\n📦 Dependency injection container information:") info(f" Total Bean count: {len(formatted_beans)}") info(f" Mock mode: {'enabled' if is_mock_mode() else 'disabled'}") diff --git a/methods/EverCore/src/core/interface/controller/base_controller.py b/methods/EverCore/src/core/interface/controller/base_controller.py index 3df551fb..fd9ffcb2 100644 --- a/methods/EverCore/src/core/interface/controller/base_controller.py +++ b/methods/EverCore/src/core/interface/controller/base_controller.py @@ -34,9 +34,9 @@ def decorator( def wrapper(func: Callable) -> Callable: # Use a special attribute to mark the function and store routing information # This avoids a global registry, making each controller self-contained - setattr(func, "__route_info__", (path, [http_method], kwargs)) + func.__route_info__ = path, [http_method], kwargs # Store extra_models for later OpenAPI generation - setattr(func, "__extra_models__", extra_models or []) + func.__extra_models__ = extra_models or [] return func return wrapper @@ -125,7 +125,7 @@ def _collect_routes(self): """ for _member_name, member in inspect.getmembers(self): if callable(member) and hasattr(member, "__route_info__"): - path, methods, route_kwargs = getattr(member, "__route_info__") + path, methods, route_kwargs = member.__route_info__ # Collect extra_models extra_models = getattr(member, "__extra_models__", []) diff --git a/methods/EverCore/src/core/lifespan/metrics_lifespan.py b/methods/EverCore/src/core/lifespan/metrics_lifespan.py index aa7a8b52..7816de64 100644 --- a/methods/EverCore/src/core/lifespan/metrics_lifespan.py +++ b/methods/EverCore/src/core/lifespan/metrics_lifespan.py @@ -41,20 +41,20 @@ async def startup(self, app: FastAPI) -> Tuple[Any, ...]: """ # Get port from environment variable or default to 9090 port = int(os.getenv("METRICS_PORT", "9090")) - + logger.info("Starting Prometheus metrics server on port %d...", port) - + try: success = start_metrics_server(port=port) - + if success: logger.info("✅ Metrics server started: %s", get_metrics_url(port=port)) app.state.metrics_port = port else: logger.warning("Metrics server already running or failed to start") - + return (port, success) - + except Exception as e: logger.error("Failed to start metrics server: %s", str(e)) # Don't raise - metrics failure shouldn't prevent app startup @@ -68,7 +68,7 @@ async def shutdown(self, app: FastAPI) -> None: app (FastAPI): FastAPI application instance """ logger.info("Metrics server will stop with main process (daemon thread)") - + # Clean up app.state if hasattr(app.state, 'metrics_port'): delattr(app.state, 'metrics_port') diff --git a/methods/EverCore/src/core/middleware/prometheus_middleware.py b/methods/EverCore/src/core/middleware/prometheus_middleware.py index 1763d6bf..0eea1fae 100644 --- a/methods/EverCore/src/core/middleware/prometheus_middleware.py +++ b/methods/EverCore/src/core/middleware/prometheus_middleware.py @@ -136,27 +136,27 @@ class PrometheusMiddleware(BaseHTTPMiddleware): app = FastAPI() app.add_middleware(PrometheusMiddleware) """ - + # Paths to skip metrics collection SKIP_PATHS = {'/metrics', '/health', '/healthz', '/ready', '/favicon.ico'} - + async def dispatch(self, request: Request, call_next: Callable) -> Response: # Skip metrics for certain paths if request.url.path in self.SKIP_PATHS: return await call_next(request) - + method = request.method - + # Record request size (before processing) request_size = 0 if request.headers.get('content-length'): request_size = int(request.headers.get('content-length', 0)) - + # Time the request start_time = time.perf_counter() status = '500' # Default to 500 in case of unhandled exception response = None - + try: response = await call_next(request) status = str(response.status_code) @@ -165,29 +165,29 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: finally: # Get path AFTER call_next - route info is now available path = _normalize_path(request) - + # Record metrics duration = time.perf_counter() - start_time - + _http_requests_total.labels( method=method, path=path, status=status, ).inc() - + _http_request_duration_seconds.labels( method=method, path=path, ).observe(duration) - + # Record request size if request_size > 0: _http_request_size_bytes.labels(method=method, path=path).observe(request_size) - + # Record response size if response and hasattr(response, 'headers') and response.headers.get('content-length'): response_size = int(response.headers.get('content-length', 0)) _http_response_size_bytes.labels(method=method, path=path).observe(response_size) - + return response diff --git a/methods/EverCore/src/core/observation/metrics/__init__.py b/methods/EverCore/src/core/observation/metrics/__init__.py index dc20e964..72ca2302 100644 --- a/methods/EverCore/src/core/observation/metrics/__init__.py +++ b/methods/EverCore/src/core/observation/metrics/__init__.py @@ -52,22 +52,22 @@ def refresh(self, labels: dict) -> float: # Counter 'Counter', 'LabeledCounter', - + # Histogram 'Histogram', 'LabeledHistogram', 'HistogramBuckets', - + # Gauge 'BaseGauge', 'LabeledGauge', - + # Registry 'get_metrics_registry', 'set_metrics_registry', 'generate_metrics_response', 'reset_metrics_registry', - + # Server 'start_metrics_server', 'is_metrics_server_running', diff --git a/methods/EverCore/src/core/observation/metrics/counter.py b/methods/EverCore/src/core/observation/metrics/counter.py index b9c7f030..923dda96 100644 --- a/methods/EverCore/src/core/observation/metrics/counter.py +++ b/methods/EverCore/src/core/observation/metrics/counter.py @@ -29,7 +29,7 @@ class Counter: # 使用 requests_total.labels(method='GET', path='/api', status='200').inc() """ - + def __init__( self, name: str, @@ -49,7 +49,7 @@ def __init__( unit: Unit (optional) """ registry = get_metrics_registry() - + self._counter = PrometheusCounter( name=name, documentation=description, @@ -61,7 +61,7 @@ def __init__( ) self._name = name self._labelnames = labelnames - + def labels(self, **labels) -> 'LabeledCounter': """ Return a Counter with labels @@ -71,7 +71,7 @@ def labels(self, **labels) -> 'LabeledCounter': """ labeled = self._counter.labels(**labels) return LabeledCounter(labeled) - + def inc(self, amount: float = 1) -> None: """ Increment counter (no labels version) @@ -84,10 +84,10 @@ def inc(self, amount: float = 1) -> None: class LabeledCounter: """Counter with labels""" - + def __init__(self, labeled_counter): self._counter = labeled_counter - + def inc(self, amount: float = 1) -> None: """ Increment counter diff --git a/methods/EverCore/src/core/observation/metrics/gauge.py b/methods/EverCore/src/core/observation/metrics/gauge.py index c3459811..863b743d 100644 --- a/methods/EverCore/src/core/observation/metrics/gauge.py +++ b/methods/EverCore/src/core/observation/metrics/gauge.py @@ -48,7 +48,7 @@ def refresh(self, labels: dict) -> float: # Usage 3: Manual set (without auto-refresh) gauge.labels(job_name='tanka').set(42) """ - + def __init__( self, name: str, @@ -69,7 +69,7 @@ def __init__( """ from .registry import get_metrics_registry registry = get_metrics_registry() - + self._gauge = PrometheusGauge( name=name, documentation=description, @@ -79,14 +79,14 @@ def __init__( unit=unit, registry=registry, ) - + self._name = name self._labelnames = labelnames - + # Store refresh tasks for each label combination # key: tuple of label values, value: RefreshTask self._refresh_tasks: Dict[Tuple, 'RefreshTask'] = {} - + def labels(self, **labels) -> 'LabeledGauge': """ Return a Gauge with labels @@ -96,26 +96,26 @@ def labels(self, **labels) -> 'LabeledGauge': """ labeled_gauge = self._gauge.labels(**labels) label_key = self._make_label_key(**labels) - + return LabeledGauge( base_gauge=self, labeled_gauge=labeled_gauge, label_key=label_key, label_dict=labels, ) - + def set(self, value: float) -> None: """Set value (no labels version)""" self._gauge.set(value) - + def inc(self, amount: float = 1) -> None: """Increment value (no labels version)""" self._gauge.inc(amount) - + def dec(self, amount: float = 1) -> None: """Decrement value (no labels version)""" self._gauge.dec(amount) - + @abstractmethod def refresh(self, labels: dict) -> float: """ @@ -142,13 +142,13 @@ def refresh(self, labels: dict) -> float: return self.queue.qsize() """ pass - + def _make_label_key(self, **labels) -> Tuple: """Generate label key""" if self._labelnames: return tuple(labels.get(name, '') for name in self._labelnames) return () - + async def _stop_all_refresh_tasks(self) -> None: """Stop all refresh tasks""" for task in self._refresh_tasks.values(): @@ -162,7 +162,7 @@ class LabeledGauge: Provides the same interface as native Gauge, with auto-refresh support. """ - + def __init__( self, base_gauge: BaseGauge, @@ -174,23 +174,23 @@ def __init__( self._labeled_gauge = labeled_gauge self._label_key = label_key self._label_dict = label_dict - + def set(self, value: float) -> None: """Set value""" self._labeled_gauge.set(value) - + def inc(self, amount: float = 1) -> None: """Increment value""" self._labeled_gauge.inc(amount) - + def dec(self, amount: float = 1) -> None: """Decrement value""" self._labeled_gauge.dec(amount) - + def set_to_current_time(self) -> None: """Set to current timestamp""" self._labeled_gauge.set_to_current_time() - + def start_refresh( self, interval_seconds: int = 5, @@ -228,11 +228,11 @@ async def refresh(self, labels: dict) -> float: ) # Schedule stop in background to avoid blocking asyncio.create_task(existing_task.stop()) - + # Create wrapper function that calls base_gauge.refresh() def refresh_wrapper(): return self._base_gauge.refresh(self._label_dict) - + # Create refresh task task = RefreshTask( refresh_func=refresh_wrapper, @@ -241,15 +241,15 @@ def refresh_wrapper(): enable_async=enable_async, label_key=self._label_key, ) - + # Store task self._base_gauge._refresh_tasks[self._label_key] = task - + # Start task task.start() - + return self - + async def stop_refresh(self) -> None: """Stop auto-refresh""" task = self._base_gauge._refresh_tasks.get(self._label_key) @@ -264,7 +264,7 @@ class RefreshTask: Each label combination has an independent refresh task. """ - + def __init__( self, refresh_func: Callable[[], float], @@ -278,31 +278,31 @@ def __init__( self.interval_seconds = interval_seconds self.enable_async = enable_async self.label_key = label_key - + self._task: Optional[asyncio.Task] = None self._running = False self._error_count = 0 - + def start(self) -> None: """Start refresh task""" if self._running: logger.warning(f"Refresh task already running for {self.label_key}") return - + self._running = True self._task = asyncio.create_task(self._refresh_loop()) logger.info( f"Started refresh task: label_key={self.label_key}, " f"interval={self.interval_seconds}s" ) - + async def stop(self) -> None: """Stop refresh task""" if not self._running: return - + self._running = False - + if self._task: self._task.cancel() try: @@ -310,28 +310,28 @@ async def stop(self) -> None: except asyncio.CancelledError: pass self._task = None - + logger.info(f"Stopped refresh task: label_key={self.label_key}") - + async def _refresh_loop(self) -> None: """Refresh loop""" while self._running: try: # Check if it's an async function if self.enable_async and ( - asyncio.iscoroutinefunction(self.refresh_func) or + asyncio.iscoroutinefunction(self.refresh_func) or inspect.iscoroutinefunction(self.refresh_func) ): value = await self.refresh_func() else: value = self.refresh_func() - + # Update Gauge self.labeled_gauge.set(value) - + # Reset error count self._error_count = 0 - + except asyncio.CancelledError: break except Exception as e: @@ -341,7 +341,7 @@ async def _refresh_loop(self) -> None: f"(error_count={self._error_count})", exc_info=True ) - + # Wait for next refresh try: await asyncio.sleep(self.interval_seconds) diff --git a/methods/EverCore/src/core/observation/metrics/histogram.py b/methods/EverCore/src/core/observation/metrics/histogram.py index 4ee1ee00..5742aa73 100644 --- a/methods/EverCore/src/core/observation/metrics/histogram.py +++ b/methods/EverCore/src/core/observation/metrics/histogram.py @@ -11,27 +11,27 @@ # Predefined bucket configurations class HistogramBuckets: """Predefined Histogram bucket configurations""" - + # Default buckets (covering 5ms - 10s) DEFAULT = ( 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0 ) - + # Fast operations (5ms - 500ms, for cache queries, simple calculations, etc.) FAST = (0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5) - + # API calls (10ms - 30s, for external API calls) # Denser buckets in 0.1-5s range for better P95/P99 accuracy API_CALL = (0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, 30.0) - + # Batch operations (100ms - 60s, for batch processing) BATCH = (0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0) - + # Embedding/Rerank (10ms - 10s, for ML inference) # Denser buckets in 0.1-3s range where most requests fall ML_INFERENCE = (0.01, 0.025, 0.05, 0.1, 0.2, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0) - + # Database queries (1ms - 5s) DATABASE = (0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0) @@ -62,7 +62,7 @@ class Histogram: with request_duration.labels(method='GET', path='/api').time(): do_something() """ - + def __init__( self, name: str, @@ -84,7 +84,7 @@ def __init__( buckets: Histogram bucket boundaries """ registry = get_metrics_registry() - + self._histogram = PrometheusHistogram( name=name, documentation=description, @@ -97,7 +97,7 @@ def __init__( ) self._name = name self._labelnames = labelnames - + def labels(self, **labels) -> 'LabeledHistogram': """ Return a Histogram with labels @@ -107,7 +107,7 @@ def labels(self, **labels) -> 'LabeledHistogram': """ labeled = self._histogram.labels(**labels) return LabeledHistogram(labeled) - + def observe(self, amount: float) -> None: """ Record an observed value (no labels version) @@ -116,7 +116,7 @@ def observe(self, amount: float) -> None: amount: Observed value """ self._histogram.observe(amount) - + def time(self): """ Return a timing context manager (no labels version) @@ -130,10 +130,10 @@ def time(self): class LabeledHistogram: """Histogram with labels""" - + def __init__(self, labeled_histogram): self._histogram = labeled_histogram - + def observe(self, amount: float) -> None: """ Record an observed value @@ -142,7 +142,7 @@ def observe(self, amount: float) -> None: amount: Observed value """ self._histogram.observe(amount) - + def time(self): """ Return a timing context manager diff --git a/methods/EverCore/src/core/observation/metrics/server.py b/methods/EverCore/src/core/observation/metrics/server.py index 186f46c2..27aad556 100644 --- a/methods/EverCore/src/core/observation/metrics/server.py +++ b/methods/EverCore/src/core/observation/metrics/server.py @@ -54,15 +54,15 @@ def start_metrics_server( # Prometheus can scrape: http://your-host:9090/metrics """ global _metrics_server_started - + if _metrics_server_started: logger.warning("Metrics server already running") return False - + # Get port from parameter, env var, or default if port is None: port = int(os.getenv("METRICS_PORT", "9090")) - + try: # Start HTTP server using prometheus_client's built-in server # This creates a daemon thread that serves /metrics endpoint @@ -71,11 +71,11 @@ def start_metrics_server( addr=addr, registry=get_metrics_registry(), ) - + _metrics_server_started = True logger.info(f"✅ Metrics server started on {addr}:{port}/metrics") return True - + except Exception as e: logger.error(f"Failed to start metrics server: {e}") return False diff --git a/methods/EverCore/src/core/oxm/es/doc_base.py b/methods/EverCore/src/core/oxm/es/doc_base.py index c10c1469..a7f504d7 100644 --- a/methods/EverCore/src/core/oxm/es/doc_base.py +++ b/methods/EverCore/src/core/oxm/es/doc_base.py @@ -78,7 +78,7 @@ def _init_date_fields_cache(cls) -> Set[str]: # Dynamically set to CustomMeta if custom_meta is not None: - setattr(custom_meta, 'date_fields', date_fields) + custom_meta.date_fields = date_fields return date_fields diff --git a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py index ee382279..b398cdcf 100644 --- a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py +++ b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py @@ -658,7 +658,7 @@ async def translate_file( async with progress_lock: progress["processed"].append(file_str) save_translation_progress(progress) - return (file_path, True, f"Skipped: file too large") + return (file_path, True, "Skipped: file too large") with open(file_path, 'r', encoding='utf-8') as f: original_content = f.read() @@ -1474,7 +1474,7 @@ def _hook_print_error_report( if files_with_cjk: files_list = list(files_with_cjk.keys()) cmd = _hook_format_translation_command(files_list) - print(f"\n1. Translate the files using:", file=sys.stderr) + print("\n1. Translate the files using:", file=sys.stderr) print(f" {cmd}", file=sys.stderr) print("\n Or for dry-run first:", file=sys.stderr) print(f" {cmd} --dry-run", file=sys.stderr) diff --git a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py index d1f3fef2..7fe04f15 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py @@ -156,7 +156,7 @@ def _parse_llm_response(self, response: str) -> Dict[str, Any]: # 5. If all fail, raise exception logger.error(f"Unable to parse LLM response: {response[:200]}...") - raise ValueError(f"Unable to parse LLM response into valid JSON format") + raise ValueError("Unable to parse LLM response into valid JSON format") async def _extract_atomic_fact( self, @@ -194,7 +194,7 @@ async def _extract_atomic_fact( # 5. Validate response format if "atomic_facts" not in data: - raise ValueError(f"Missing 'atomic_facts' field in LLM response") + raise ValueError("Missing 'atomic_facts' field in LLM response") atomic_fact_data = data["atomic_facts"] @@ -270,7 +270,7 @@ async def extract_atomic_fact( except Exception as e: logger.warning(f"Retrying to extract atomic fact {retry+1}/5: {e}") if retry == 4: - logger.error(f"Failed to extract atomic fact after 5 retries") + logger.error("Failed to extract atomic fact after 5 retries") return None continue diff --git a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py index d71892d7..8856beff 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py @@ -156,7 +156,7 @@ async def generate_foresights_for_conversation( except Exception as e: logger.warning(f"Foresight generation retry {retry+1}/5: {e}") if retry == 4: - logger.error(f"Foresight generation failed after 5 retries") + logger.error("Foresight generation failed after 5 retries") return [] continue From e2152cb638daf1784a9967c61cef147187abe35a Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 18:55:47 +0800 Subject: [PATCH 06/24] Remove unused imports flagged by ruff F401 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical cleanup via `ruff check --select F401 --fix --unsafe-fixes`. 138 of 139 unused imports removed across 67 files. Audited removals before committing: - DI decorators (`from core.di.decorators import service / component / repository`) were imported in a few adapter modules but never used as `@service` etc. `core.di.decorators` is a pure decorator factory module with no module-load side effects, so removing the dangling imports is safe. - stdlib (`os`, `time`, `json`, `random`, `uuid`, `numpy`, `jieba`, ...) and typing helpers (`Optional`, `List`, `Tuple`, ...) form the bulk of removals. - `__init__.py` files are intentionally exempt (per the `[tool.ruff.lint.per-file-ignores]` rule allowing re-exports). The one remaining F401 (`pyinstrument` in profile_middleware.py) is an availability-probe import — `try: import pyinstrument` — and is suppressed with `# noqa: F401` since the import is the side effect. Net impact: ruff baseline 387 -> 244 errors. ty count unchanged at 891 (F401 doesn't affect type inference). All src/*.py files compile successfully. --- methods/EverCore/src/agentic_layer/agentic_utils.py | 2 +- methods/EverCore/src/agentic_layer/memory_manager.py | 11 +---------- .../src/agentic_layer/profile_search_service.py | 2 +- .../EverCore/src/agentic_layer/search_mem_service.py | 1 - .../EverCore/src/agentic_layer/vectorize_deepinfra.py | 1 - methods/EverCore/src/agentic_layer/vectorize_vllm.py | 1 - methods/EverCore/src/api_specs/dtos/memory.py | 4 +--- methods/EverCore/src/app.py | 1 - methods/EverCore/src/base_app.py | 1 - methods/EverCore/src/biz_layer/mem_db_operations.py | 3 --- methods/EverCore/src/biz_layer/mem_memorize.py | 7 ------- methods/EverCore/src/biz_layer/mem_sync.py | 3 +-- methods/EverCore/src/common_utils/cli_ui.py | 2 +- .../addonize/tests/test_addon_bean_order_strategy.py | 1 - .../EverCore/src/core/component/config_provider.py | 3 +-- .../component/llm/llm_adapter/anthropic_adapter.py | 1 - .../src/core/component/llm/llm_adapter/completion.py | 2 +- .../core/component/llm/llm_adapter/gemini_adapter.py | 3 --- .../src/core/component/milvus_client_factory.py | 1 - .../src/core/component/openai_compatible_client.py | 1 - methods/EverCore/src/core/context/context_manager.py | 2 +- methods/EverCore/src/core/di/container.py | 3 --- methods/EverCore/src/core/di/decorators.py | 1 - methods/EverCore/src/core/di/exceptions.py | 2 +- methods/EverCore/src/core/di/scanner.py | 1 - .../src/core/di/tests/test_bean_order_strategy.py | 1 - .../EverCore/src/core/di/tests/test_di_container.py | 7 +------ methods/EverCore/src/core/di/tests/test_di_scanner.py | 5 ----- .../EverCore/src/core/lifespan/business_lifespan.py | 2 +- .../EverCore/src/core/lifespan/database_lifespan.py | 1 - .../EverCore/src/core/lifespan/metrics_lifespan.py | 2 +- .../src/core/longjob/recycle_consumer_base.py | 1 - .../src/core/middleware/app_logic_middleware.py | 2 +- .../core/middleware/database_session_middleware.py | 1 - .../src/core/middleware/profile_middleware.py | 4 ++-- methods/EverCore/src/core/observation/logger.py | 3 +-- .../src/core/observation/tracing/decorators.py | 2 +- methods/EverCore/src/core/oxm/es/doc_base.py | 1 - methods/EverCore/src/core/oxm/es/migration/utils.py | 2 +- .../EverCore/src/core/oxm/milvus/base_repository.py | 3 +-- .../EverCore/src/core/oxm/mongo/migration/manager.py | 1 - methods/EverCore/src/core/oxm/pg/base_repository.py | 3 +-- .../redis_group_queue/kafka_consumer_record_item.py | 1 - methods/EverCore/src/core/tenants/tenant_router.py | 2 +- .../tenants/tenantize/kv/redis/tenant_key_utils.py | 2 -- .../tenantize/oxm/es/tenant_field_es_interceptor.py | 5 +---- .../sensitive_info/sensitive_info_tool.py | 1 - .../adapters/input/api/memory/memory_controller.py | 7 +------ .../out/persistence/document/memory/agent_case.py | 2 +- .../out/persistence/document/memory/agent_skill.py | 2 +- .../persistence/document/memory/atomic_fact_record.py | 2 +- .../persistence/document/memory/episodic_memory.py | 2 +- .../out/persistence/document/memory/mem_scene.py | 1 - .../out/persistence/document/memory/user_profile.py | 3 +-- .../repository/agent_skill_raw_repository.py | 1 - .../persistence/repository/memcell_raw_repository.py | 3 +-- .../elasticsearch/converter/atomic_fact_converter.py | 3 --- .../elasticsearch/converter/foresight_converter.py | 1 - .../repository/user_profile_milvus_repository.py | 1 - .../src/memory_layer/cluster_manager/manager.py | 1 - .../memcell_extractor/base_memcell_extractor.py | 2 +- .../memory_extractor/agent_case_extractor.py | 1 - .../memory_extractor/atomic_fact_extractor.py | 2 +- .../memory_extractor/episode_memory_extractor.py | 6 ++---- .../memory_extractor/foresight_extractor.py | 4 ++-- methods/EverCore/src/memory_layer/memory_manager.py | 4 ---- .../memory_layer/profile_indexer/profile_indexer.py | 2 +- 67 files changed, 37 insertions(+), 125 deletions(-) diff --git a/methods/EverCore/src/agentic_layer/agentic_utils.py b/methods/EverCore/src/agentic_layer/agentic_utils.py index 68870461..d895e896 100644 --- a/methods/EverCore/src/agentic_layer/agentic_utils.py +++ b/methods/EverCore/src/agentic_layer/agentic_utils.py @@ -11,7 +11,7 @@ import json import asyncio import logging -from typing import List, Tuple, Optional, Dict, Any +from typing import List, Tuple, Dict, Any from dataclasses import dataclass logger = logging.getLogger(__name__) diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index 22d8f9c6..ddfc7108 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -4,15 +4,12 @@ import logging import asyncio -from datetime import datetime, timedelta +from datetime import datetime import jieba -import numpy as np import time from dataclasses import dataclass -from api_specs import memory_types from api_specs.memory_types import ( - BaseMemory, EpisodeMemory, AtomicFact, Foresight, @@ -59,17 +56,11 @@ from core.nlp.stopwords_utils import filter_stopwords from common_utils.datetime_utils import ( from_iso_format, - get_now_with_timezone, - to_iso_format, ) from infra_layer.adapters.out.persistence.repository.memcell_raw_repository import ( MemCellRawRepository, ) from service.raw_message_service import RawMessageService -from infra_layer.adapters.out.persistence.document.memory.memcell import DataTypeEnum -from infra_layer.adapters.out.persistence.document.memory.user_profile import ( - UserProfile, -) from infra_layer.adapters.out.search.repository.episodic_memory_milvus_repository import ( EpisodicMemoryMilvusRepository, ) diff --git a/methods/EverCore/src/agentic_layer/profile_search_service.py b/methods/EverCore/src/agentic_layer/profile_search_service.py index 3260489c..54e30b0e 100644 --- a/methods/EverCore/src/agentic_layer/profile_search_service.py +++ b/methods/EverCore/src/agentic_layer/profile_search_service.py @@ -10,7 +10,7 @@ """ import os -from typing import List, Dict, Any, Optional +from typing import Dict, Any, Optional import time from core.di import get_bean_by_type diff --git a/methods/EverCore/src/agentic_layer/search_mem_service.py b/methods/EverCore/src/agentic_layer/search_mem_service.py index 9c25f919..92f3966e 100644 --- a/methods/EverCore/src/agentic_layer/search_mem_service.py +++ b/methods/EverCore/src/agentic_layer/search_mem_service.py @@ -22,7 +22,6 @@ from typing import Any, Dict, List, Optional from core.di import service, get_bean -from api_specs.dtos.memory import SearchAtomicFactItem from core.nlp.stopwords_utils import filter_stopwords from core.observation.stage_timer import timed, timed_parallel from api_specs.dtos.memory import ( diff --git a/methods/EverCore/src/agentic_layer/vectorize_deepinfra.py b/methods/EverCore/src/agentic_layer/vectorize_deepinfra.py index 7a3b6f68..9b943e6e 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_deepinfra.py +++ b/methods/EverCore/src/agentic_layer/vectorize_deepinfra.py @@ -4,7 +4,6 @@ Commercial API implementation for DeepInfra embedding service """ -import os import logging from typing import Optional, Tuple from dataclasses import dataclass diff --git a/methods/EverCore/src/agentic_layer/vectorize_vllm.py b/methods/EverCore/src/agentic_layer/vectorize_vllm.py index d67f64d8..553a8805 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_vllm.py +++ b/methods/EverCore/src/agentic_layer/vectorize_vllm.py @@ -5,7 +5,6 @@ such as vLLM, Ollama, or other OpenAI-compatible endpoints. """ -import os import logging from typing import Optional, Tuple from dataclasses import dataclass diff --git a/methods/EverCore/src/api_specs/dtos/memory.py b/methods/EverCore/src/api_specs/dtos/memory.py index c1bafbc9..d4a300a8 100644 --- a/methods/EverCore/src/api_specs/dtos/memory.py +++ b/methods/EverCore/src/api_specs/dtos/memory.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union import json import re @@ -25,7 +25,6 @@ field_validator, model_validator, SkipValidation, - SerializeAsAny, ) from api_specs.dtos.base import BaseApiResponse @@ -35,7 +34,6 @@ Metadata, QueryMetadata, RetrieveMethod, - MessageSenderRole, ) from core.oxm.constants import MAGIC_ALL, MAX_RETRIEVE_LIMIT from biz_layer.retrieve_constants import MAX_GROUP_IDS_COUNT diff --git a/methods/EverCore/src/app.py b/methods/EverCore/src/app.py index fa445369..32f5cebe 100644 --- a/methods/EverCore/src/app.py +++ b/methods/EverCore/src/app.py @@ -9,7 +9,6 @@ from core.capability.app_capability import ApplicationCapability from core.observation.logger import get_logger from core.interface.controller.base_controller import BaseController -from core.middleware.user_context_middleware import UserContextMiddleware from core.middleware.app_logic_middleware import AppLogicMiddleware from core.middleware.prometheus_middleware import PrometheusMiddleware from fastapi.middleware import Middleware diff --git a/methods/EverCore/src/base_app.py b/methods/EverCore/src/base_app.py index e183ba29..edd15d3c 100644 --- a/methods/EverCore/src/base_app.py +++ b/methods/EverCore/src/base_app.py @@ -10,7 +10,6 @@ from fastapi.middleware.cors import CORSMiddleware from core.observation.logger import get_logger -from core.middleware.database_session_middleware import DatabaseSessionMiddleware from core.middleware.global_exception_handler import global_exception_handler from core.middleware.profile_middleware import ProfileMiddleware from core.di.utils import get_bean_by_type diff --git a/methods/EverCore/src/biz_layer/mem_db_operations.py b/methods/EverCore/src/biz_layer/mem_db_operations.py index dcb9d3a8..b6cb5985 100644 --- a/methods/EverCore/src/biz_layer/mem_db_operations.py +++ b/methods/EverCore/src/biz_layer/mem_db_operations.py @@ -9,7 +9,6 @@ 4. Status table operation functions: Manage the lifecycle of conversation status """ -import time from api_specs.dtos import MemorizeRequest from api_specs.memory_types import MemCell, RawDataType from core.di import get_bean_by_type @@ -71,7 +70,6 @@ def _convert_episode_memory_to_doc( from infra_layer.adapters.out.persistence.document.memory.episodic_memory import ( EpisodicMemory, ) - from agentic_layer.vectorize_service import get_vectorize_service # Parse timestamp to datetime object if current_time is None: @@ -260,7 +258,6 @@ def _convert_agent_case_to_doc( # ==================== Database Operation Functions ==================== -from core.observation.tracing.decorators import trace_logger async def _save_memcell_to_database( diff --git a/methods/EverCore/src/biz_layer/mem_memorize.py b/methods/EverCore/src/biz_layer/mem_memorize.py index 2376f6f7..8e095063 100644 --- a/methods/EverCore/src/biz_layer/mem_memorize.py +++ b/methods/EverCore/src/biz_layer/mem_memorize.py @@ -1,7 +1,5 @@ from dataclasses import dataclass -import random import time -import json import traceback from core.observation.stage_timer import timed, timed_parallel @@ -16,7 +14,6 @@ from api_specs.memory_types import ( MemoryType, MemCell, - BaseMemory, EpisodeMemory, RawDataType, Foresight, @@ -45,9 +42,7 @@ ConversationDataRepository, ) from typing import List, Dict, Optional, Any -import uuid from datetime import datetime, timedelta -import os import asyncio from collections import defaultdict from common_utils.datetime_utils import get_now_with_timezone, to_iso_format @@ -399,7 +394,6 @@ async def _trigger_profile_extraction( ) from memory_layer.llm.llm_provider import build_default_provider from core.di import get_bean_by_type - import os total_memcell_count = sum( mem_scene_state.cluster_counts.get(cid, 0) for cid in cluster_ids @@ -719,7 +713,6 @@ async def _trigger_agent_skill_extraction( _update_status_for_continuing_conversation, _update_status_after_memcell_extraction, ) -from typing import Tuple def if_memorize(memcell: MemCell) -> bool: diff --git a/methods/EverCore/src/biz_layer/mem_sync.py b/methods/EverCore/src/biz_layer/mem_sync.py index 33f4dd36..de819f9e 100644 --- a/methods/EverCore/src/biz_layer/mem_sync.py +++ b/methods/EverCore/src/biz_layer/mem_sync.py @@ -3,7 +3,7 @@ Responsible for writing unified foresight and atomic facts into Milvus / Elasticsearch. """ -from typing import Optional, List, Dict, Any +from typing import Optional, List, Dict import logging from datetime import datetime @@ -38,7 +38,6 @@ AtomicFactEsRepository, ) from core.di import get_bean_by_type, service -from common_utils.datetime_utils import get_now_with_timezone logger = logging.getLogger(__name__) diff --git a/methods/EverCore/src/common_utils/cli_ui.py b/methods/EverCore/src/common_utils/cli_ui.py index da845223..0cb21ce7 100644 --- a/methods/EverCore/src/common_utils/cli_ui.py +++ b/methods/EverCore/src/common_utils/cli_ui.py @@ -35,7 +35,7 @@ import shutil import sys import unicodedata -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import List, Optional, Sequence # ============================================================================ diff --git a/methods/EverCore/src/core/addons/addonize/tests/test_addon_bean_order_strategy.py b/methods/EverCore/src/core/addons/addonize/tests/test_addon_bean_order_strategy.py index c3ce145d..20099f3a 100644 --- a/methods/EverCore/src/core/addons/addonize/tests/test_addon_bean_order_strategy.py +++ b/methods/EverCore/src/core/addons/addonize/tests/test_addon_bean_order_strategy.py @@ -10,7 +10,6 @@ import os import pytest -from typing import Set, Type from core.di.bean_definition import BeanDefinition, BeanScope from core.addons.addonize.addon_bean_order_strategy import AddonBeanOrderStrategy diff --git a/methods/EverCore/src/core/component/config_provider.py b/methods/EverCore/src/core/component/config_provider.py index a34738f9..9f6809dd 100644 --- a/methods/EverCore/src/core/component/config_provider.py +++ b/methods/EverCore/src/core/component/config_provider.py @@ -1,7 +1,6 @@ -import os import yaml import json -from typing import Dict, Any, Optional +from typing import Dict, Any from core.di.decorators import component from common_utils.project_path import CURRENT_DIR diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py b/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py index 346aa209..80e25541 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py @@ -4,7 +4,6 @@ from typing import Dict, Any, List, Union, AsyncGenerator import os import httpx -from core.di.decorators import service from core.component.llm.llm_adapter.completion import ( ChatCompletionRequest, diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/completion.py b/methods/EverCore/src/core/component/llm/llm_adapter/completion.py index 108ab420..17ef99d0 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/completion.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/completion.py @@ -1,5 +1,5 @@ from typing import Dict, Any, List, Optional -from dataclasses import dataclass, field +from dataclasses import dataclass from pydantic import BaseModel from core.component.llm.llm_adapter.message import ChatMessage diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py index 547776b6..b5ce597b 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py @@ -4,12 +4,9 @@ from typing import Dict, Any, List, Union, AsyncGenerator import os from google.genai.client import Client -from core.di.decorators import service from google.genai.types import ( GenerateContentConfig, ContentDict, - HarmCategory, - HarmBlockThreshold, ) from google.genai.types import ThinkingConfig from core.component.llm.llm_adapter.completion import ( diff --git a/methods/EverCore/src/core/component/milvus_client_factory.py b/methods/EverCore/src/core/component/milvus_client_factory.py index 092da61a..121a415d 100644 --- a/methods/EverCore/src/core/component/milvus_client_factory.py +++ b/methods/EverCore/src/core/component/milvus_client_factory.py @@ -7,7 +7,6 @@ import os import asyncio from typing import Optional, Dict -from hashlib import md5 from pymilvus import MilvusClient from core.di.decorators import component diff --git a/methods/EverCore/src/core/component/openai_compatible_client.py b/methods/EverCore/src/core/component/openai_compatible_client.py index 98e0cdf4..768f553d 100644 --- a/methods/EverCore/src/core/component/openai_compatible_client.py +++ b/methods/EverCore/src/core/component/openai_compatible_client.py @@ -1,5 +1,4 @@ import asyncio -import os from typing import Dict, Any, List, Optional, AsyncGenerator, Union from core.di.decorators import component diff --git a/methods/EverCore/src/core/context/context_manager.py b/methods/EverCore/src/core/context/context_manager.py index 4546b546..a8f9b8c1 100644 --- a/methods/EverCore/src/core/context/context_manager.py +++ b/methods/EverCore/src/core/context/context_manager.py @@ -1,5 +1,5 @@ from contextvars import copy_context, Context -from typing import Optional, Dict, Any, Callable, TypeVar, Coroutine, Union, Tuple +from typing import Optional, Dict, Any, Callable, TypeVar, Coroutine from functools import wraps from sqlmodel.ext.asyncio.session import AsyncSession diff --git a/methods/EverCore/src/core/di/container.py b/methods/EverCore/src/core/di/container.py index 9f5e6ad1..fcdd1e3d 100644 --- a/methods/EverCore/src/core/di/container.py +++ b/methods/EverCore/src/core/di/container.py @@ -18,9 +18,7 @@ Optional, Any, List, - Set, Callable, - Union, get_origin, get_args, ) @@ -35,7 +33,6 @@ DuplicateBeanError, FactoryError, DependencyResolutionError, - MockNotEnabledError, ) T = TypeVar('T') diff --git a/methods/EverCore/src/core/di/decorators.py b/methods/EverCore/src/core/di/decorators.py index d978234e..68804f35 100644 --- a/methods/EverCore/src/core/di/decorators.py +++ b/methods/EverCore/src/core/di/decorators.py @@ -4,7 +4,6 @@ """ from typing import Type, TypeVar, Optional, Callable, Any, Dict -from functools import wraps from core.di.container import get_container from core.di.bean_definition import BeanScope diff --git a/methods/EverCore/src/core/di/exceptions.py b/methods/EverCore/src/core/di/exceptions.py index 4411b6c5..5ae1e30c 100644 --- a/methods/EverCore/src/core/di/exceptions.py +++ b/methods/EverCore/src/core/di/exceptions.py @@ -3,7 +3,7 @@ Dependency injection system exception class definitions """ -from typing import Type, Any, List +from typing import Type, List class DIException(Exception): diff --git a/methods/EverCore/src/core/di/scanner.py b/methods/EverCore/src/core/di/scanner.py index 4e9be1eb..2a30b108 100644 --- a/methods/EverCore/src/core/di/scanner.py +++ b/methods/EverCore/src/core/di/scanner.py @@ -3,7 +3,6 @@ Component Scanner """ -import os import sys import importlib from pathlib import Path diff --git a/methods/EverCore/src/core/di/tests/test_bean_order_strategy.py b/methods/EverCore/src/core/di/tests/test_bean_order_strategy.py index 18ac4822..1a3bd3ca 100644 --- a/methods/EverCore/src/core/di/tests/test_bean_order_strategy.py +++ b/methods/EverCore/src/core/di/tests/test_bean_order_strategy.py @@ -12,7 +12,6 @@ """ import pytest -from typing import Set, Type from core.di.bean_definition import BeanDefinition, BeanScope from core.di.bean_order_strategy import BeanOrderStrategy diff --git a/methods/EverCore/src/core/di/tests/test_di_container.py b/methods/EverCore/src/core/di/tests/test_di_container.py index 2425eb8d..d414912a 100644 --- a/methods/EverCore/src/core/di/tests/test_di_container.py +++ b/methods/EverCore/src/core/di/tests/test_di_container.py @@ -9,8 +9,7 @@ """ import pytest -from abc import ABC, abstractmethod -from typing import List +from abc import ABC from core.di.container import DIContainer from core.di.bean_definition import BeanScope from core.di.exceptions import BeanNotFoundError @@ -20,7 +19,6 @@ MySQLUserRepository, PostgreSQLUserRepository, MockUserRepository, - UserService, UserServiceImpl, # Notification service related NotificationService, @@ -28,9 +26,6 @@ SMSNotificationService, PushNotificationService, # Email service related - EmailService, - SMTPEmailService, - # Database connection related DatabaseConnection, create_database_connection, create_readonly_connection, diff --git a/methods/EverCore/src/core/di/tests/test_di_scanner.py b/methods/EverCore/src/core/di/tests/test_di_scanner.py index a24a982f..90a4ae45 100644 --- a/methods/EverCore/src/core/di/tests/test_di_scanner.py +++ b/methods/EverCore/src/core/di/tests/test_di_scanner.py @@ -9,9 +9,6 @@ """ import pytest -import tempfile -import shutil -from pathlib import Path from core.di.container import DIContainer, get_container from core.di.scanner import ComponentScanner from core.di.decorators import component, service, repository, mock_impl, factory @@ -19,8 +16,6 @@ from core.di.tests.test_fixtures import ( UserRepository, MySQLUserRepository, - NotificationService, - EmailNotificationService, ) diff --git a/methods/EverCore/src/core/lifespan/business_lifespan.py b/methods/EverCore/src/core/lifespan/business_lifespan.py index 61fe9258..a40eb5b2 100644 --- a/methods/EverCore/src/core/lifespan/business_lifespan.py +++ b/methods/EverCore/src/core/lifespan/business_lifespan.py @@ -6,7 +6,7 @@ from typing import Dict, Any from core.observation.logger import get_logger -from core.di.utils import get_bean_by_type, get_beans_by_type, get_bean +from core.di.utils import get_bean_by_type, get_beans_by_type from core.di.decorators import component from core.interface.controller.base_controller import BaseController from core.capability.app_capability import ApplicationCapability diff --git a/methods/EverCore/src/core/lifespan/database_lifespan.py b/methods/EverCore/src/core/lifespan/database_lifespan.py index 12861e39..31ed0781 100644 --- a/methods/EverCore/src/core/lifespan/database_lifespan.py +++ b/methods/EverCore/src/core/lifespan/database_lifespan.py @@ -7,7 +7,6 @@ from core.observation.logger import get_logger from core.di.utils import get_bean_by_type -from core.di.decorators import component from core.component.database_connection_provider import DatabaseConnectionProvider from .lifespan_interface import LifespanProvider diff --git a/methods/EverCore/src/core/lifespan/metrics_lifespan.py b/methods/EverCore/src/core/lifespan/metrics_lifespan.py index 7816de64..85a74c36 100644 --- a/methods/EverCore/src/core/lifespan/metrics_lifespan.py +++ b/methods/EverCore/src/core/lifespan/metrics_lifespan.py @@ -9,7 +9,7 @@ from core.observation.logger import get_logger from core.di.decorators import component -from core.observation.metrics import start_metrics_server, is_metrics_server_running, get_metrics_url +from core.observation.metrics import start_metrics_server, get_metrics_url from .lifespan_interface import LifespanProvider logger = get_logger(__name__) diff --git a/methods/EverCore/src/core/longjob/recycle_consumer_base.py b/methods/EverCore/src/core/longjob/recycle_consumer_base.py index 89453b1a..4284942f 100644 --- a/methods/EverCore/src/core/longjob/recycle_consumer_base.py +++ b/methods/EverCore/src/core/longjob/recycle_consumer_base.py @@ -8,7 +8,6 @@ import random from abc import ABC, abstractmethod from typing import Optional, Any, Dict -from datetime import datetime from core.longjob.interfaces import ( LongJobInterface, diff --git a/methods/EverCore/src/core/middleware/app_logic_middleware.py b/methods/EverCore/src/core/middleware/app_logic_middleware.py index 2f8b2379..9b35d73a 100644 --- a/methods/EverCore/src/core/middleware/app_logic_middleware.py +++ b/methods/EverCore/src/core/middleware/app_logic_middleware.py @@ -3,7 +3,7 @@ Responsible for extracting and setting application-level context information, and handling application-related logic (e.g., reporting) """ -from typing import Callable, Dict, Any, Optional +from typing import Callable, Optional from fastapi import Request, HTTPException from starlette.responses import Response diff --git a/methods/EverCore/src/core/middleware/database_session_middleware.py b/methods/EverCore/src/core/middleware/database_session_middleware.py index 4d0273c7..e3f7b69a 100644 --- a/methods/EverCore/src/core/middleware/database_session_middleware.py +++ b/methods/EverCore/src/core/middleware/database_session_middleware.py @@ -9,7 +9,6 @@ from core.context.context import ( set_current_session, clear_current_session, - get_current_session, ) from core.component.database_session_provider import DatabaseSessionProvider from core.di.utils import get_bean_by_type diff --git a/methods/EverCore/src/core/middleware/profile_middleware.py b/methods/EverCore/src/core/middleware/profile_middleware.py index 0eeb426a..f50dbd8d 100644 --- a/methods/EverCore/src/core/middleware/profile_middleware.py +++ b/methods/EverCore/src/core/middleware/profile_middleware.py @@ -20,7 +20,7 @@ """ import os -from typing import Callable, Optional +from typing import Callable from fastapi import Request from fastapi.responses import HTMLResponse @@ -59,7 +59,7 @@ def __init__(self, app: ASGIApp): self._profiler_available = False if self._profiling_enabled: try: - import pyinstrument + import pyinstrument # noqa: F401 # availability probe self._profiler_available = True logger.info("✅ Performance profiling middleware enabled") diff --git a/methods/EverCore/src/core/observation/logger.py b/methods/EverCore/src/core/observation/logger.py index 89855ab8..d5b5c1aa 100644 --- a/methods/EverCore/src/core/observation/logger.py +++ b/methods/EverCore/src/core/observation/logger.py @@ -1,11 +1,10 @@ import logging import traceback -from typing import Any, Optional +from typing import Optional from enum import Enum from functools import lru_cache import sys import os -from datetime import datetime from core.context.context import get_current_app_info diff --git a/methods/EverCore/src/core/observation/tracing/decorators.py b/methods/EverCore/src/core/observation/tracing/decorators.py index 472d01fd..0c2e12b4 100644 --- a/methods/EverCore/src/core/observation/tracing/decorators.py +++ b/methods/EverCore/src/core/observation/tracing/decorators.py @@ -5,7 +5,7 @@ """ from functools import wraps -from typing import Any, Dict, Callable, Optional +from typing import Callable, Optional import logging import time diff --git a/methods/EverCore/src/core/oxm/es/doc_base.py b/methods/EverCore/src/core/oxm/es/doc_base.py index a7f504d7..688f180e 100644 --- a/methods/EverCore/src/core/oxm/es/doc_base.py +++ b/methods/EverCore/src/core/oxm/es/doc_base.py @@ -1,7 +1,6 @@ import typing from typing import Type, Any, Dict, Set -import os from fnmatch import fnmatch from datetime import datetime from elasticsearch.dsl import MetaField, AsyncDocument, field as e_field diff --git a/methods/EverCore/src/core/oxm/es/migration/utils.py b/methods/EverCore/src/core/oxm/es/migration/utils.py index bc8be6cb..0557a564 100644 --- a/methods/EverCore/src/core/oxm/es/migration/utils.py +++ b/methods/EverCore/src/core/oxm/es/migration/utils.py @@ -11,7 +11,7 @@ from elasticsearch.dsl import AsyncDocument from core.observation.logger import get_logger from core.di.utils import get_all_subclasses -from core.oxm.es.doc_base import DocBase, get_index_ns +from core.oxm.es.doc_base import DocBase from core.oxm.es.es_utils import is_abstract_doc_class logger = get_logger(__name__) diff --git a/methods/EverCore/src/core/oxm/milvus/base_repository.py b/methods/EverCore/src/core/oxm/milvus/base_repository.py index cfa49e67..3522e9e1 100644 --- a/methods/EverCore/src/core/oxm/milvus/base_repository.py +++ b/methods/EverCore/src/core/oxm/milvus/base_repository.py @@ -5,12 +5,11 @@ """ from abc import ABC -from typing import Optional, TypeVar, Generic, Type, List, Any +from typing import Optional, TypeVar, Generic, Type, List from pymilvus.orm.mutation import MutationResult from core.oxm.milvus.milvus_collection_base import MilvusCollectionBase from core.oxm.milvus.async_collection import AsyncCollection from core.observation.logger import get_logger -from core.di.utils import get_bean logger = get_logger(__name__) diff --git a/methods/EverCore/src/core/oxm/mongo/migration/manager.py b/methods/EverCore/src/core/oxm/mongo/migration/manager.py index 72637371..b3d61f7e 100644 --- a/methods/EverCore/src/core/oxm/mongo/migration/manager.py +++ b/methods/EverCore/src/core/oxm/mongo/migration/manager.py @@ -9,7 +9,6 @@ import logging import subprocess import sys -from datetime import datetime from pathlib import Path from typing import Optional diff --git a/methods/EverCore/src/core/oxm/pg/base_repository.py b/methods/EverCore/src/core/oxm/pg/base_repository.py index 52e8f0df..a8e98a14 100644 --- a/methods/EverCore/src/core/oxm/pg/base_repository.py +++ b/methods/EverCore/src/core/oxm/pg/base_repository.py @@ -1,7 +1,6 @@ from abc import ABC, abstractmethod -from typing import List, Optional, TypeVar, Generic, Type +from typing import List, Optional, TypeVar, Generic from core.oxm.pg.audit_base import get_auditable_model -from core.di.decorators import repository # Define generic type T = TypeVar('T', bound=get_auditable_model()) diff --git a/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py b/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py index 2cb604e1..ddca40cf 100644 --- a/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py +++ b/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py @@ -5,7 +5,6 @@ Uses BSON format to handle binary data, ensuring data integrity """ -import json import base64 from typing import Optional, Sequence, Tuple, Any, Dict from dataclasses import dataclass diff --git a/methods/EverCore/src/core/tenants/tenant_router.py b/methods/EverCore/src/core/tenants/tenant_router.py index bf5dcfab..06f68753 100644 --- a/methods/EverCore/src/core/tenants/tenant_router.py +++ b/methods/EverCore/src/core/tenants/tenant_router.py @@ -17,7 +17,7 @@ """ from abc import ABC, abstractmethod -from typing import Any, Dict, Optional +from typing import Optional from fastapi import Request diff --git a/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py b/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py index 87fdc74f..cbedb3b4 100644 --- a/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py +++ b/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py @@ -4,9 +4,7 @@ Provides tenant isolation for Redis key names by prepending the tenant ID to achieve multi-tenant data isolation. """ -from typing import Optional -from core.tenants.tenant_contextvar import get_current_tenant_id def build_tenant_redis_key(prefix: str, tenant_id: str, key: str) -> str: diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py b/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py index 04b6d095..c05e49e0 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py @@ -32,10 +32,8 @@ Collection, Dict, FrozenSet, - List, Mapping, Optional, - Set, Tuple, Union, ) @@ -45,10 +43,9 @@ ApiResponseMeta, AsyncTransport, HeadApiResponse, - HttpHeaders, ObjectApiResponse, ) -from elastic_transport._models import DEFAULT, DefaultType, SniffOptions +from elastic_transport._models import DEFAULT, DefaultType from elastic_transport._otel import OpenTelemetrySpan from core.observation.logger import get_logger diff --git a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py index f9d1220b..a24405d0 100644 --- a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py +++ b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py @@ -28,7 +28,6 @@ from pathlib import Path from dataclasses import dataclass, field from enum import Enum -from typing import Optional # ============================================================================== # Path Configuration diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py index a9ae63b9..640ac910 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py @@ -11,21 +11,17 @@ """ import asyncio -import json import logging import time -from contextlib import suppress from fastapi import HTTPException, Request as FastAPIRequest from core.di.decorators import controller from core.di import get_bean_by_type -from core.interface.controller.base_controller import BaseController, get, post +from core.interface.controller.base_controller import BaseController, post from core.observation.stage_timer import stage_timed, timed -from core.constants.errors import ErrorCode, ErrorStatus from agentic_layer.memory_manager import MemoryManager from api_specs.request_converter import ( - convert_dict_to_retrieve_mem_request, convert_personal_add_to_memorize_request, convert_group_add_to_memorize_request, convert_personal_flush_to_memorize_request, @@ -56,7 +52,6 @@ from service.raw_message_service import RawMessageService from service.sender_service import SenderService from service.memcell_delete_service import MemCellDeleteService -from api_specs.memory_types import RawDataType from agentic_layer.metrics.memorize_metrics import ( record_memorize_request, record_memorize_error, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_case.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_case.py index 3cdb1273..e9932623 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_case.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_case.py @@ -6,7 +6,7 @@ """ from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import List, Optional from core.oxm.mongo.document_base import DocumentBase from core.tenants.tenantize.oxm.mongo.tenant_aware_document import ( TenantAwareDocumentBaseWithSoftDelete, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_skill.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_skill.py index b95bc3f7..d68370dc 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_skill.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/agent_skill.py @@ -6,7 +6,7 @@ """ from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import List, Optional from core.oxm.mongo.document_base import DocumentBase from core.tenants.tenantize.oxm.mongo.tenant_aware_document import ( TenantAwareDocumentBaseWithSoftDelete, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/atomic_fact_record.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/atomic_fact_record.py index 651b37e3..b664f514 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/atomic_fact_record.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/atomic_fact_record.py @@ -5,7 +5,7 @@ """ from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import List, Optional from core.oxm.mongo.document_base import DocumentBase from core.tenants.tenantize.oxm.mongo.tenant_aware_document import ( TenantAwareDocumentBaseWithSoftDelete, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/episodic_memory.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/episodic_memory.py index b032c1c9..9fc4130c 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/episodic_memory.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/episodic_memory.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import List, Optional from core.oxm.mongo.document_base import DocumentBase from core.tenants.tenantize.oxm.mongo.tenant_aware_document import ( TenantAwareDocumentBaseWithSoftDelete, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/mem_scene.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/mem_scene.py index c43f7482..1f2c337c 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/mem_scene.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/mem_scene.py @@ -1,4 +1,3 @@ -from datetime import datetime from typing import List, Optional, Dict, Any from core.tenants.tenantize.oxm.mongo.tenant_aware_document import ( TenantAwareDocumentBase, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/user_profile.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/user_profile.py index 59f1fbf6..207b3151 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/user_profile.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/document/memory/user_profile.py @@ -1,5 +1,4 @@ -from datetime import datetime -from typing import List, Optional, Dict, Any +from typing import Optional, Dict, Any from core.tenants.tenantize.oxm.mongo.tenant_aware_document import ( TenantAwareDocumentBaseWithSoftDelete, ) diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py index f253c3a7..c231db99 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py @@ -7,7 +7,6 @@ from typing import List, Optional, Dict, Any from pymongo.asynchronous.client_session import AsyncClientSession -from bson import ObjectId from core.observation.logger import get_logger from core.di.decorators import repository from core.oxm.mongo.base_repository import BaseRepository diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py index 337467a8..d9ad9c8d 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py @@ -9,7 +9,7 @@ from typing import List, Optional, Dict, Any, Type from bson import ObjectId from pydantic import BaseModel -from beanie.operators import And, GTE, LT, Eq, RegEx, Or +from beanie.operators import And, GTE, LT, Eq, Or from pymongo.asynchronous.client_session import AsyncClientSession from core.observation.logger import get_logger from core.di.decorators import repository @@ -18,7 +18,6 @@ from infra_layer.adapters.out.persistence.document.memory.memcell import ( MemCell, - DataTypeEnum, ) logger = get_logger(__name__) diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/atomic_fact_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/atomic_fact_converter.py index db8d5bd5..9a82a339 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/atomic_fact_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/atomic_fact_converter.py @@ -4,11 +4,8 @@ Converts MongoDB v1_atomic_fact_records to ES v1_atomic_fact_record. """ -from typing import List -import jieba from core.oxm.es.base_converter import BaseEsConverter from core.observation.logger import get_logger -from core.nlp.stopwords_utils import filter_stopwords from infra_layer.adapters.out.search.elasticsearch.memory.atomic_fact import ( AtomicFactDoc, ) diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/foresight_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/foresight_converter.py index 71b4e28a..120244f1 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/foresight_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/elasticsearch/converter/foresight_converter.py @@ -4,7 +4,6 @@ Converts MongoDB v1_foresight_records to ES v1_foresight_record. """ -from typing import List from core.oxm.es.base_converter import BaseEsConverter from core.observation.logger import get_logger from infra_layer.adapters.out.search.elasticsearch.memory.foresight import ForesightDoc diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py b/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py index addb40ee..2b2b84c4 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py @@ -5,7 +5,6 @@ Only maps search-essential fields. Full data retrieved from MongoDB using id. """ -from datetime import datetime from typing import List, Optional, Dict, Any from core.oxm.milvus.base_repository import BaseMilvusRepository from core.oxm.constants import MAGIC_ALL diff --git a/methods/EverCore/src/memory_layer/cluster_manager/manager.py b/methods/EverCore/src/memory_layer/cluster_manager/manager.py index 0f807cb9..307ff2da 100644 --- a/methods/EverCore/src/memory_layer/cluster_manager/manager.py +++ b/methods/EverCore/src/memory_layer/cluster_manager/manager.py @@ -14,7 +14,6 @@ import json import numpy as np from typing import Any, Callable, Dict, List, Optional, Tuple -from pathlib import Path from memory_layer.cluster_manager.config import ClusterManagerConfig from core.observation.logger import get_logger diff --git a/methods/EverCore/src/memory_layer/memcell_extractor/base_memcell_extractor.py b/methods/EverCore/src/memory_layer/memcell_extractor/base_memcell_extractor.py index 0917ffb1..34902b71 100644 --- a/methods/EverCore/src/memory_layer/memcell_extractor/base_memcell_extractor.py +++ b/methods/EverCore/src/memory_layer/memcell_extractor/base_memcell_extractor.py @@ -7,7 +7,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import List, Dict, Any, Optional +from typing import List, Optional from memory_layer.llm.llm_provider import LLMProvider from api_specs.memory_types import RawDataType, BaseMemory, MemCell from api_specs.dtos import RawData diff --git a/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py index 7936699c..15afd267 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py @@ -33,7 +33,6 @@ ) from memory_layer.prompts import get_prompt_by from api_specs.memory_types import ( - MemCell, RawDataType, AgentCase, get_text_from_content_items, diff --git a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py index 7fe04f15..86980b7f 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py @@ -5,7 +5,7 @@ Each extraction result contains a time and a list of atomic facts extracted from the episode. """ -from typing import Optional, List, Dict, Any +from typing import Optional, Dict, Any from datetime import datetime import json import re diff --git a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py index cfd32d81..3bd148a2 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py @@ -5,10 +5,10 @@ from boundary detection results (BoundaryResult). """ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Optional, List, Dict, Any from datetime import datetime -import re, json, asyncio, uuid +import re, json from memory_layer.prompts import get_prompt_by @@ -22,8 +22,6 @@ MemoryType, EpisodeMemory, RawDataType, - MemCell, - ParentType, get_text_from_content_items, ) diff --git a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py index 8856beff..c05fb8ea 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py @@ -4,7 +4,7 @@ """ import json -from typing import List, Dict, Any, Optional +from typing import List, Optional from datetime import datetime, timedelta from memory_layer.prompts import get_prompt_by @@ -13,7 +13,7 @@ MemoryExtractor, MemoryExtractRequest, ) -from api_specs.memory_types import MemoryType, MemCell, Foresight, BaseMemory +from api_specs.memory_types import MemoryType, Foresight, BaseMemory from agentic_layer.vectorize_service import get_vectorize_service from core.observation.logger import get_logger from core.observation.stage_timer import timed diff --git a/methods/EverCore/src/memory_layer/memory_manager.py b/methods/EverCore/src/memory_layer/memory_manager.py index beec525d..cbebf481 100644 --- a/methods/EverCore/src/memory_layer/memory_manager.py +++ b/methods/EverCore/src/memory_layer/memory_manager.py @@ -1,7 +1,4 @@ -from dataclasses import dataclass -from datetime import datetime import time -import asyncio from typing import List, Optional, Dict, Any from core.observation.logger import get_logger @@ -33,7 +30,6 @@ ) from memory_layer.memory_extractor.episode_memory_extractor import ( EpisodeMemoryExtractor, - EpisodeMemoryExtractRequest, ) from memory_layer.memory_extractor.profile_extractor import ( ProfileExtractor, diff --git a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py index 94765730..075d73d0 100644 --- a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py +++ b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py @@ -15,7 +15,7 @@ 4. Prevents race conditions in distributed environment """ -from typing import List, Dict, Any, Optional +from typing import List, Dict, Optional from core.di import get_bean_by_type from core.di.decorators import service From 362e02134ae90fc594b9343432eaa18596775788 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 19:01:15 +0800 Subject: [PATCH 07/24] Replace blocking calls in async functions (ASYNC230/251) Fixes 5 ruff ASYNC violations. The first is a real bug; the others are dev-script cleanup for consistency now that aiofiles is already a project dependency. - core/oxm/es/migration/utils.py:177,182: `wait_for_task_completion` polls Elasticsearch task status inside an async function but used `time.sleep(5)` and `time.sleep(10)` on error, freezing the entire event loop on each iteration. ES rebuilds can run for minutes-to-hours, so this would stall every other coroutine in the process (FastAPI lifespan, background tasks, etc.). Switch to `await asyncio.sleep(...)`. - devops_scripts/i18n/i18n_tool.py:663,701 and devops_scripts/sensitive_info/sensitive_info_tool.py:477: Replace synchronous `with open(...)` / `f.read()` / `f.write()` with `async with aiofiles.open(...)` / `await f.read()` / `await f.write()`. These are dev-tool scripts where blocking IO impact is negligible, but the conversion is mechanical and removes the noise from future lint reports. --- methods/EverCore/src/core/oxm/es/migration/utils.py | 6 +++--- methods/EverCore/src/devops_scripts/i18n/i18n_tool.py | 9 +++++---- .../devops_scripts/sensitive_info/sensitive_info_tool.py | 7 +++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/methods/EverCore/src/core/oxm/es/migration/utils.py b/methods/EverCore/src/core/oxm/es/migration/utils.py index 0557a564..184aca85 100644 --- a/methods/EverCore/src/core/oxm/es/migration/utils.py +++ b/methods/EverCore/src/core/oxm/es/migration/utils.py @@ -4,7 +4,7 @@ Provides generic Elasticsearch index rebuilding and migration functionality. """ -import time +import asyncio import traceback from typing import Type, Any from elasticsearch import NotFoundError, RequestError @@ -174,12 +174,12 @@ async def wait_for_task_completion(es_connect: Any, task_id: str) -> None: "Rebuild progress: %d/%d (%.1f%%)", created, total, progress ) - time.sleep(5) # Check every 5 seconds + await asyncio.sleep(5) # Check every 5 seconds except (NotFoundError, RequestError) as e: traceback.print_exc() logger.error("Failed to check task status: %s", e) - time.sleep(10) # Wait longer when error occurs + await asyncio.sleep(10) # Wait longer when error occurs async def update_aliases( diff --git a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py index b398cdcf..6ba3a8ac 100644 --- a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py +++ b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py @@ -44,6 +44,7 @@ import asyncio import json import subprocess +import aiofiles from fnmatch import fnmatch from pathlib import Path from typing import Optional, TYPE_CHECKING @@ -660,8 +661,8 @@ async def translate_file( save_translation_progress(progress) return (file_path, True, "Skipped: file too large") - with open(file_path, 'r', encoding='utf-8') as f: - original_content = f.read() + async with aiofiles.open(file_path, 'r', encoding='utf-8') as f: + original_content = await f.read() if not contains_chinese(original_content): print(f"{progress_prefix} [SKIP] {file_path} - No Chinese text found") @@ -698,8 +699,8 @@ async def translate_file( return (file_path, False, error_msg) if not dry_run: - with open(file_path, 'w', encoding='utf-8') as f: - f.write(translated_content) + async with aiofiles.open(file_path, 'w', encoding='utf-8') as f: + await f.write(translated_content) print(f"{progress_prefix} [DONE] {file_path}") else: print(f"{progress_prefix} [DRY-RUN] {file_path} - Would translate") diff --git a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py index a24405d0..5e4aa0e3 100644 --- a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py +++ b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py @@ -24,6 +24,7 @@ import sys import asyncio import json +import aiofiles from fnmatch import fnmatch from pathlib import Path from dataclasses import dataclass, field @@ -474,8 +475,10 @@ async def analyze_file_with_ai( # Read file content try: - with open(file_path, "r", encoding="utf-8", errors="ignore") as f: - content = f.read() + async with aiofiles.open( + file_path, "r", encoding="utf-8", errors="ignore" + ) as f: + content = await f.read() except Exception as e: return FileCheckResult(file_path=file_path, error=str(e)) From dabc3c4cf977f386fadbeeec7b87f1d63f76ac09 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 19:06:52 +0800 Subject: [PATCH 08/24] Apply ruff --unsafe-fixes for whitespace, F841, dict literals, etc. Mechanical cleanup of 190 ruff violations across 39 files. Manually audited the auto-fixes; the noteworthy decisions: - W293/W291 inside src/memory_layer/prompts/* was REVERTED. Those triple-quoted strings are LLM prompt templates and the trailing whitespace is part of the literal sent to the model. Three files restored to HEAD: prompts/__init__.py, en/atomic_fact_prompts.py, zh/conv_prompts.py. - F841 "unused variable" auto-fix mostly deleted assignments where the RHS is pure (literals, len(), sorted(), str()). For cases where ruff conservatively kept the RHS as an orphan expression (`get_tenant_aware_collection_name(origin_alias_name)` etc.), manually verified those functions are pure and deleted the expression lines too: - test_di_scanner.py:284 (`len(container._named_beans)`) - tenant_aware_collection_with_suffix.py:350 - user_profile_milvus_converter.py:52 The `job = await pool.enqueue_job(...)` case correctly kept the awaited call -- only the unused name binding was removed. - E731 lambda-to-def conversion in milvus_rebuild_collection.py produced wrong inner indentation; fixed to PEP 8 4-space. - C408 dict(...) -> {...}, B905 zip(..., strict=False), E401 multiple-imports-on-one-line, and the remaining W291/W293 outside prompts were applied verbatim. Result: ruff 239 -> 56 errors. All src/ files compile. ty 891 -> 839 (F841 deletions remove type tracking on those vars). --- .../src/agentic_layer/memory_manager.py | 38 ++++++++-------- .../agentic_layer/metrics/memorize_metrics.py | 6 +-- .../agentic_layer/metrics/rerank_metrics.py | 18 ++++---- .../agentic_layer/metrics/retrieve_metrics.py | 6 +-- .../metrics/vectorize_metrics.py | 12 ++--- .../agentic_layer/profile_search_service.py | 14 +++--- .../src/agentic_layer/rerank_service.py | 2 +- .../src/agentic_layer/retrieval_utils.py | 14 +++--- .../src/agentic_layer/vectorize_service.py | 2 +- .../src/agentic_layer/vectorize_vllm.py | 2 +- .../EverCore/src/biz_layer/mem_memorize.py | 4 +- methods/EverCore/src/bootstrap.py | 2 +- .../src/core/asynctasks/task_manager.py | 2 +- .../llm/tokenizer/tokenizer_factory.py | 16 +++---- .../src/core/di/tests/test_di_scanner.py | 1 - methods/EverCore/src/core/di/utils.py | 2 +- .../src/core/lock/redis_distributed_lock.py | 14 +++--- .../src/core/longjob/recycle_consumer_base.py | 2 +- .../core/middleware/prometheus_middleware.py | 8 ++-- .../src/core/observation/metrics/__init__.py | 14 +++--- .../src/core/observation/metrics/counter.py | 14 +++--- .../src/core/observation/metrics/gauge.py | 40 ++++++++--------- .../src/core/observation/metrics/histogram.py | 20 ++++----- .../src/core/observation/metrics/registry.py | 10 ++--- .../src/core/observation/metrics/server.py | 18 ++++---- .../src/core/oxm/mongo/base_repository.py | 2 +- .../src/core/oxm/mongo/migration/manager.py | 18 ++++---- .../redis_group_queue_lua_scripts.py | 44 +++++++++---------- .../tenant_aware_collection_with_suffix.py | 1 - .../data_fix/milvus_rebuild_collection.py | 7 +-- .../sensitive_info/sensitive_info_tool.py | 20 ++++----- .../converter/agent_skill_milvus_converter.py | 1 - .../user_profile_milvus_converter.py | 1 - methods/EverCore/src/manage.py | 4 +- .../conv_memcell_extractor.py | 4 +- .../episode_memory_extractor.py | 4 +- .../src/memory_layer/memory_manager.py | 10 ++--- .../profile_indexer/profile_indexer.py | 2 +- .../src/service/memcell_delete_service.py | 16 +++---- 39 files changed, 207 insertions(+), 208 deletions(-) diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index ddfc7108..5ddb0296 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -447,7 +447,7 @@ async def retrieve_mem_keyword( """Keyword-based memory retrieval""" top_k = retrieve_mem_request.top_k is_unlimited_mode = top_k == -1 - memory_type = ( + ( retrieve_mem_request.memory_types[0].value if retrieve_mem_request.memory_types else 'unknown' @@ -577,7 +577,7 @@ async def retrieve_mem_vector( """Vector-based memory retrieval""" top_k = retrieve_mem_request.top_k is_unlimited_mode = top_k == -1 - memory_type = ( + ( retrieve_mem_request.memory_types[0].value if retrieve_mem_request.memory_types else 'unknown' @@ -791,7 +791,7 @@ async def retrieve_mem_hybrid( self, retrieve_mem_request: 'RetrieveMemRequest' ) -> RetrieveMemResponse: """Hybrid memory retrieval: keyword + vector + rerank""" - memory_type = ( + ( retrieve_mem_request.memory_types[0].value if retrieve_mem_request.memory_types else 'unknown' @@ -1305,7 +1305,7 @@ async def group_by_groupid_stratagy( atomic_fact = fields['atomic_fact'] foresight = fields['foresight'] evidence = fields['evidence'] - extend_data = fields['extend_data'] + fields['extend_data'] search_source = fields['search_source'] # Process timestamp timestamp = from_iso_format(timestamp_raw) @@ -1321,21 +1321,21 @@ async def group_by_groupid_stratagy( logger.debug(f"Memcell not found: event_id={parent_id}") # Create object based on memory type - base_kwargs = dict( - id=hit_id, - memory_type=memory_type_value, - user_id=user_id, - timestamp=timestamp, - group_id=group_id, - participants=participants, - sender_ids=sender_ids, - parent_type=parent_type, - parent_id=parent_id, - type=RawDataType.from_string(event_type), - score=score, - original_data=original_data, - extend={'_search_source': search_source}, - ) + base_kwargs = { + "id": hit_id, + "memory_type": memory_type_value, + "user_id": user_id, + "timestamp": timestamp, + "group_id": group_id, + "participants": participants, + "sender_ids": sender_ids, + "parent_type": parent_type, + "parent_id": parent_id, + "type": RawDataType.from_string(event_type), + "score": score, + "original_data": original_data, + "extend": {'_search_source': search_source}, + } match memory_type_value: case MemoryType.ATOMIC_FACT.value: diff --git a/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py b/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py index e1251b76..5915ceff 100644 --- a/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py @@ -15,7 +15,7 @@ record_extraction_stage, get_space_id_for_metrics, ) - + # Record successful memorize request record_memorize_request( space_id=get_space_id_for_metrics(), @@ -23,7 +23,7 @@ status='success', duration_seconds=0.5, ) - + # Record extraction stage duration record_extraction_stage( space_id=get_space_id_for_metrics(), @@ -217,7 +217,7 @@ def get_raw_data_type_label(raw_data_type: Optional[str]) -> str: Labels: - space_id: Tenant space identifier - raw_data_type: Type of raw data (conversation, etc.) -- stage: init_state, extract_episodes, extract_foresights, extract_atomic_facts, +- stage: init_state, extract_episodes, extract_foresights, extract_atomic_facts, update_memcell_cluster, process_memories Buckets: 10ms - 10s for ML inference diff --git a/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py b/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py index 071724ea..49f03603 100644 --- a/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py @@ -11,18 +11,18 @@ RERANK_FALLBACK_TOTAL, RERANK_ERRORS_TOTAL, ) - + # Record successful rerank request RERANK_REQUESTS_TOTAL.labels( provider='vllm', status='success' ).inc() - + # Record duration RERANK_DURATION_SECONDS.labels( provider='vllm' ).observe(0.234) - + # Record documents count RERANK_DOCUMENTS_TOTAL.labels( provider='vllm' @@ -137,13 +137,13 @@ def record_rerank_request( ) -> None: """ Helper function to record all rerank metrics in one call - + Args: provider: Service provider (vllm, deepinfra) status: Request status (success, error, timeout, fallback) duration_seconds: Operation duration in seconds documents_count: Number of documents reranked - + Example: record_rerank_request( provider='vllm', @@ -176,12 +176,12 @@ def record_rerank_fallback( ) -> None: """ Helper function to record rerank fallback event - + Args: primary_provider: Primary provider that failed fallback_provider: Fallback provider used reason: Fallback reason (error, timeout, max_failures_exceeded) - + Example: record_rerank_fallback( primary_provider='vllm', @@ -202,11 +202,11 @@ def record_rerank_error( ) -> None: """ Helper function to record rerank error - + Args: provider: Service provider error_type: Error type (api_error, timeout, rate_limit, validation_error, unknown) - + Example: record_rerank_error( provider='vllm', diff --git a/methods/EverCore/src/agentic_layer/metrics/retrieve_metrics.py b/methods/EverCore/src/agentic_layer/metrics/retrieve_metrics.py index cfb71a9d..ced90d3d 100644 --- a/methods/EverCore/src/agentic_layer/metrics/retrieve_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/retrieve_metrics.py @@ -14,20 +14,20 @@ RETRIEVE_STAGE_DURATION_SECONDS, RETRIEVE_ERRORS_TOTAL, ) - + # Record successful retrieval RETRIEVE_REQUESTS_TOTAL.labels( memory_type='episodic_memory', retrieve_method='vector', status='success' ).inc() - + # Record duration RETRIEVE_DURATION_SECONDS.labels( memory_type='episodic_memory', retrieve_method='vector_search' ).observe(0.567) - + # Record stage-specific duration RETRIEVE_STAGE_DURATION_SECONDS.labels( stage='milvus_search', diff --git a/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py b/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py index 200987d6..2ea610a6 100644 --- a/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py @@ -134,7 +134,7 @@ def record_vectorize_request( ) -> None: """ Helper function to record all vectorize metrics in one call - + Args: provider: Service provider (vllm, deepinfra) operation: Operation type (get_embedding, get_embeddings, get_embeddings_batch) @@ -142,7 +142,7 @@ def record_vectorize_request( duration_seconds: Operation duration in seconds batch_size: Number of texts processed tokens: Number of tokens processed (optional, for cost tracking) - + Example: record_vectorize_request( provider='vllm', @@ -185,12 +185,12 @@ def record_vectorize_fallback( ) -> None: """ Helper function to record vectorize fallback event - + Args: primary_provider: Primary provider that failed fallback_provider: Fallback provider used reason: Fallback reason (error, timeout, max_failures_exceeded) - + Example: record_vectorize_fallback( primary_provider='vllm', @@ -212,12 +212,12 @@ def record_vectorize_error( ) -> None: """ Helper function to record vectorize error - + Args: provider: Service provider operation: Operation type error_type: Error type (api_error, timeout, rate_limit, validation_error, unknown) - + Example: record_vectorize_error( provider='vllm', diff --git a/methods/EverCore/src/agentic_layer/profile_search_service.py b/methods/EverCore/src/agentic_layer/profile_search_service.py index 54e30b0e..5ba7a1f0 100644 --- a/methods/EverCore/src/agentic_layer/profile_search_service.py +++ b/methods/EverCore/src/agentic_layer/profile_search_service.py @@ -31,15 +31,15 @@ def parse_embed_text(embed_text: str, item_type: str) -> Dict[str, str]: """ Parse embed_text to extract category/trait_name and description. - + Format: - explicit_info: "category: description" - implicit_trait: "trait_name: description. basis" or "trait_name: description" - + Args: embed_text: The embedded text string item_type: "explicit_info" or "implicit_trait" - + Returns: Dict with parsed fields """ @@ -79,7 +79,7 @@ def parse_embed_text(embed_text: str, item_type: str) -> Dict[str, str]: class ProfileSearchService: """ Profile Search Service - + Searches user profile items in Milvus using vector similarity. No reranking step - directly returns Milvus results with score threshold. """ @@ -89,7 +89,7 @@ def __init__( milvus_repo: Optional[UserProfileMilvusRepository] = None, ): """Initialize service - + Args: milvus_repo: User profile Milvus repository (auto-injected if None) """ @@ -112,14 +112,14 @@ async def search_profiles( ) -> Dict[str, Any]: """ Search profile items by query text - + Args: query: Search query text user_id: User ID filter group_id: Group ID filter top_k: Maximum number of results score_threshold: Minimum similarity score (0.0-1.0) - + Returns: Dict with: - profiles: List of profile items diff --git a/methods/EverCore/src/agentic_layer/rerank_service.py b/methods/EverCore/src/agentic_layer/rerank_service.py index e668e8c2..404ad570 100644 --- a/methods/EverCore/src/agentic_layer/rerank_service.py +++ b/methods/EverCore/src/agentic_layer/rerank_service.py @@ -6,7 +6,7 @@ Usage: from agentic_layer.rerank_service import get_rerank_service - + service = get_rerank_service() result = await service.rerank_memories(query, hits, top_k) """ diff --git a/methods/EverCore/src/agentic_layer/retrieval_utils.py b/methods/EverCore/src/agentic_layer/retrieval_utils.py index 955611d8..e1667557 100644 --- a/methods/EverCore/src/agentic_layer/retrieval_utils.py +++ b/methods/EverCore/src/agentic_layer/retrieval_utils.py @@ -28,7 +28,7 @@ def build_bm25_index(candidates): from nltk.stem import PorterStemmer from nltk.tokenize import word_tokenize from rank_bm25 import BM25Okapi - except ImportError as e: + except ImportError: return None, None, None, None # Ensure NLTK data is downloaded @@ -106,7 +106,7 @@ async def search_with_bm25( scores = bm25.get_scores(tokenized_query) # Sort and return Top-K - results = sorted(zip(candidates, scores), key=lambda x: x[1], reverse=True)[:top_k] + results = sorted(zip(candidates, scores, strict=False), key=lambda x: x[1], reverse=True)[:top_k] return results @@ -119,14 +119,14 @@ def reciprocal_rank_fusion( doc_map = {} # Process first result set - for rank, (doc, score) in enumerate(results1, start=1): + for rank, (doc, _score) in enumerate(results1, start=1): doc_id = doc.get('id') if doc_id not in doc_map: doc_map[doc_id] = doc doc_rrf_scores[doc_id] = doc_rrf_scores.get(doc_id, 0.0) + 1.0 / (k + rank) # Process second result set - for rank, (doc, score) in enumerate(results2, start=1): + for rank, (doc, _score) in enumerate(results2, start=1): doc_id = doc.get('id') if doc_id not in doc_map: doc_map[doc_id] = doc @@ -170,7 +170,7 @@ def vector_anchored_fusion( Returns: Fused results [(doc_id, score), ...] sorted by score descending. """ - vec_score_map: Dict[str, float] = {doc_id: s for doc_id, s in vector_results} + vec_score_map: Dict[str, float] = dict(vector_results) # Saturate BM25 scores into [0, 1) kw_sat_map: Dict[str, float] = {} @@ -241,7 +241,7 @@ async def lightweight_retrieval( continue emb_results = sorted(scores, key=lambda x: x[1], reverse=True)[:emb_top_n] - except Exception as e: + except Exception: pass metadata["emb_count"] = len(emb_results) @@ -322,7 +322,7 @@ def multi_rrf_fusion(results_list: List[List[Tuple]], k: int = 60) -> List[Tuple # Iterate through each query's retrieval results for query_results in results_list: - for rank, (doc, score) in enumerate(query_results, start=1): + for rank, (doc, _score) in enumerate(query_results, start=1): doc_id = id(doc) if doc_id not in doc_map: doc_map[doc_id] = doc diff --git a/methods/EverCore/src/agentic_layer/vectorize_service.py b/methods/EverCore/src/agentic_layer/vectorize_service.py index 0a2efe83..7b94e4b4 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_service.py +++ b/methods/EverCore/src/agentic_layer/vectorize_service.py @@ -6,7 +6,7 @@ Usage: from agentic_layer.vectorize_service import get_vectorize_service - + service = get_vectorize_service() embedding = await service.get_embedding("Hello world") # Auto-fallback """ diff --git a/methods/EverCore/src/agentic_layer/vectorize_vllm.py b/methods/EverCore/src/agentic_layer/vectorize_vllm.py index 553a8805..9ed79a59 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_vllm.py +++ b/methods/EverCore/src/agentic_layer/vectorize_vllm.py @@ -32,7 +32,7 @@ class VllmVectorizeConfig: class VllmVectorizeService(BaseVectorizeService): """ vLLM self-deployed embedding service implementation - + Supports: - vLLM (https://github.com/vllm-project/vllm) - Any OpenAI-compatible embedding endpoint diff --git a/methods/EverCore/src/biz_layer/mem_memorize.py b/methods/EverCore/src/biz_layer/mem_memorize.py index 8e095063..34c6d22e 100644 --- a/methods/EverCore/src/biz_layer/mem_memorize.py +++ b/methods/EverCore/src/biz_layer/mem_memorize.py @@ -978,7 +978,7 @@ def _process_episode_results(state: ExtractionState, results: List[Any]): # Personal Episodes if not state.is_solo_scene: - for user_id, result in zip(state.participants, results[1:]): + for user_id, result in zip(state.participants, results[1:], strict=False): if isinstance(result, Exception): logger.error( f"[MemCell Processing] ❌ Personal Episode exception: user_id={user_id}" @@ -1190,7 +1190,7 @@ async def _save_episodes( saved_map = await save_memory_docs(payloads) saved_docs = saved_map.get(MemoryType.EPISODIC_MEMORY, []) - for ep, saved_doc in zip(episodic_source, saved_docs): + for ep, saved_doc in zip(episodic_source, saved_docs, strict=False): ep.id = str(saved_doc.id) state.parent_docs_map[str(saved_doc.id)] = saved_doc diff --git a/methods/EverCore/src/bootstrap.py b/methods/EverCore/src/bootstrap.py index 5b21698c..6f510a1a 100644 --- a/methods/EverCore/src/bootstrap.py +++ b/methods/EverCore/src/bootstrap.py @@ -138,7 +138,7 @@ async def async_main(): python src/bootstrap.py tests/algorithms/debug_my_model.py python src/bootstrap.py unit_test/memory_manager_single_test.py --verbose python src/bootstrap.py evaluation/dynamic_memory_evaluation/locomo_eval.py --dataset small - + Environment variables: MOCK_MODE=true Enable Mock mode (for testing) """, diff --git a/methods/EverCore/src/core/asynctasks/task_manager.py b/methods/EverCore/src/core/asynctasks/task_manager.py index 4626f18b..6005ac73 100644 --- a/methods/EverCore/src/core/asynctasks/task_manager.py +++ b/methods/EverCore/src/core/asynctasks/task_manager.py @@ -331,7 +331,7 @@ async def enqueue_task( defer_until = get_now_with_timezone() + timedelta(seconds=delay) # Enqueue task - job = await pool.enqueue_job( + await pool.enqueue_job( task_name, task_context, *args, diff --git a/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py b/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py index ef1698e8..70aa1933 100644 --- a/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py +++ b/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py @@ -25,7 +25,7 @@ class TokenizerFactory: """ Tokenizer Factory - + Provides tokenizer caching and management functionality. Cache key format: "{provider}:{encoding_name}" (e.g., "tiktoken:o200k_base") """ @@ -38,13 +38,13 @@ def __init__(self): def get_tokenizer_from_tiktoken(self, encoding_name: str) -> tiktoken.Encoding: """ Get a tiktoken tokenizer by encoding name, with caching. - + Args: encoding_name: The name of the tiktoken encoding (e.g., "o200k_base", "cl100k_base") - + Returns: tiktoken.Encoding: The tokenizer instance - + Example: >>> tokenizer = factory.get_tokenizer_from_tiktoken("o200k_base") >>> tokens = tokenizer.encode("Hello, world!") @@ -61,10 +61,10 @@ def get_tokenizer_from_tiktoken(self, encoding_name: str) -> tiktoken.Encoding: def load_default_encodings(self) -> None: """ Preload default tiktoken encodings during application startup. - + This method should be called during application lifespan startup to ensure tokenizers are ready before handling requests. - + The encodings loaded are defined in DEFAULT_TIKTOKEN_ENCODINGS. """ logger.info("Preloading %d tiktoken encodings...", len(DEFAULT_TIKTOKEN_ENCODINGS)) @@ -81,7 +81,7 @@ def load_default_encodings(self) -> None: def get_cached_tokenizer_count(self) -> int: """ Get the number of cached tokenizers. - + Returns: int: Number of tokenizers currently in cache """ @@ -90,7 +90,7 @@ def get_cached_tokenizer_count(self) -> int: def clear_cache(self) -> None: """ Clear the tokenizer cache. - + This is mainly useful for testing purposes. """ self._tokenizers.clear() diff --git a/methods/EverCore/src/core/di/tests/test_di_scanner.py b/methods/EverCore/src/core/di/tests/test_di_scanner.py index 90a4ae45..b9d9447f 100644 --- a/methods/EverCore/src/core/di/tests/test_di_scanner.py +++ b/methods/EverCore/src/core/di/tests/test_di_scanner.py @@ -281,7 +281,6 @@ class TestConditionalRegistration: def test_lazy_registration(self): """Test lazy registration""" container = get_container() - initial_bean_count = len(container._named_beans) # Define component with lazy registration @component(name="lazy_comp_unique_1", lazy=True) diff --git a/methods/EverCore/src/core/di/utils.py b/methods/EverCore/src/core/di/utils.py index 74a29532..35174a6c 100644 --- a/methods/EverCore/src/core/di/utils.py +++ b/methods/EverCore/src/core/di/utils.py @@ -269,7 +269,7 @@ def get_or_create(bean_type: Type[T], factory: Callable[[], T] = None) -> T: instance = bean_type() register_bean(bean_type, instance) return instance - except Exception as e: + except Exception: raise BeanNotFoundError(bean_type=bean_type) diff --git a/methods/EverCore/src/core/lock/redis_distributed_lock.py b/methods/EverCore/src/core/lock/redis_distributed_lock.py index c47e978c..051d1d60 100644 --- a/methods/EverCore/src/core/lock/redis_distributed_lock.py +++ b/methods/EverCore/src/core/lock/redis_distributed_lock.py @@ -116,13 +116,13 @@ class RedisDistributedLockManager: local lock_key = KEYS[1] local owner_id = ARGV[1] local timeout_ms = tonumber(ARGV[2]) - + -- Get current lock information -- Note: When lock_key does not exist, HMGET returns {false, false} local lock_info = redis.call('HMGET', lock_key, 'owner', 'count') local current_owner = lock_info[1] -- false when not exists local current_count = tonumber(lock_info[2]) or 0 -- tonumber(false) is nil, use 0 as default - + if current_owner == false or current_owner == owner_id then -- Lock is not occupied (current_owner == false) or held by current coroutine, can acquire/reenter local new_count = current_count + 1 @@ -141,18 +141,18 @@ class RedisDistributedLockManager: LUA_RELEASE_SCRIPT = """ local lock_key = KEYS[1] local owner_id = ARGV[1] - + -- Get current lock information -- Note: When lock_key does not exist, HMGET returns {false, false} local lock_info = redis.call('HMGET', lock_key, 'owner', 'count') local current_owner = lock_info[1] -- false when not exists local current_count = tonumber(lock_info[2]) or 0 -- tonumber(false) is nil, use 0 as default - + if current_owner ~= owner_id then -- Not the lock holder, cannot release or lock does not exist return 0 end - + local new_count = current_count - 1 if new_count <= 0 then -- Reentry count reaches zero, completely release the lock @@ -169,13 +169,13 @@ class RedisDistributedLockManager: LUA_STATUS_SCRIPT = """ local lock_key = KEYS[1] local owner_id = ARGV[1] - + -- Get current lock information -- Note: When lock_key does not exist, HMGET returns {false, false} local lock_info = redis.call('HMGET', lock_key, 'owner', 'count') local current_owner = lock_info[1] -- false when not exists local current_count = tonumber(lock_info[2]) or 0 -- tonumber(false) is nil, use 0 as default - + if current_owner == false then return {0, 0} -- Not locked elseif current_owner == owner_id then diff --git a/methods/EverCore/src/core/longjob/recycle_consumer_base.py b/methods/EverCore/src/core/longjob/recycle_consumer_base.py index 4284942f..bc1247aa 100644 --- a/methods/EverCore/src/core/longjob/recycle_consumer_base.py +++ b/methods/EverCore/src/core/longjob/recycle_consumer_base.py @@ -265,7 +265,7 @@ async def _consume_messages(self) -> None: exc_info=True, ) - except Exception as e: + except Exception: # Other exceptions will be caught and handled in the outer loop raise diff --git a/methods/EverCore/src/core/middleware/prometheus_middleware.py b/methods/EverCore/src/core/middleware/prometheus_middleware.py index 0eea1fae..1f2f2f88 100644 --- a/methods/EverCore/src/core/middleware/prometheus_middleware.py +++ b/methods/EverCore/src/core/middleware/prometheus_middleware.py @@ -7,7 +7,7 @@ Usage: from fastapi import FastAPI from core.middleware.prometheus_middleware import PrometheusMiddleware - + app = FastAPI() app.add_middleware(PrometheusMiddleware) """ @@ -120,18 +120,18 @@ def _normalize_path(request: Request) -> str: class PrometheusMiddleware(BaseHTTPMiddleware): """ Prometheus HTTP Metrics Middleware - + Automatically records: - http_requests_total (Counter): Total requests by method, path, status - http_request_duration_seconds (Histogram): Request latency - http_request_size_bytes (Histogram): Request body size - http_response_size_bytes (Histogram): Response body size - + Design inspired by: - Kratos (Bilibili): middleware.Middleware pattern - Hertz (ByteDance): promhttp.InstrumentHandler pattern - Go kit: endpoint.Middleware pattern - + Usage: app = FastAPI() app.add_middleware(PrometheusMiddleware) diff --git a/methods/EverCore/src/core/observation/metrics/__init__.py b/methods/EverCore/src/core/observation/metrics/__init__.py index 72ca2302..e571313a 100644 --- a/methods/EverCore/src/core/observation/metrics/__init__.py +++ b/methods/EverCore/src/core/observation/metrics/__init__.py @@ -5,29 +5,29 @@ Usage: from core.observation.metrics import Counter, Histogram, BaseGauge - + # Counter - monotonically increasing counter requests_total = Counter('http_requests_total', 'Total requests', ['method']) requests_total.labels(method='GET').inc() - + # Histogram - distribution statistics of observed values request_duration = Histogram( - 'http_request_duration_seconds', - 'Request duration', + 'http_request_duration_seconds', + 'Request duration', ['method'], buckets=HistogramBuckets.API_CALL ) request_duration.labels(method='GET').observe(0.123) - + # BaseGauge - instantaneous value with auto-refresh (inheritance) class QueueSizeGauge(BaseGauge): def __init__(self, queue): super().__init__('queue_size', 'Queue size', ['queue_name']) self.queue = queue - + def refresh(self, labels: dict) -> float: return self.queue.qsize() - + # Using Gauge gauge = QueueSizeGauge(queue) gauge.labels(queue_name='main').start_refresh() # default 5 second refresh diff --git a/methods/EverCore/src/core/observation/metrics/counter.py b/methods/EverCore/src/core/observation/metrics/counter.py index 923dda96..13e328b7 100644 --- a/methods/EverCore/src/core/observation/metrics/counter.py +++ b/methods/EverCore/src/core/observation/metrics/counter.py @@ -11,21 +11,21 @@ class Counter: """ Counter metric wrapper - + Features: - Monotonically increasing counter (can only increment) - Suitable for total requests, total errors, etc. - Business code does not need to import prometheus_client directly - + Usage: from core.observation.metrics import Counter - + requests_total = Counter( name='http_requests_total', description='Total HTTP requests', labelnames=['method', 'path', 'status'] ) - + # 使用 requests_total.labels(method='GET', path='/api', status='200').inc() """ @@ -65,7 +65,7 @@ def __init__( def labels(self, **labels) -> 'LabeledCounter': """ Return a Counter with labels - + Returns: LabeledCounter instance """ @@ -75,7 +75,7 @@ def labels(self, **labels) -> 'LabeledCounter': def inc(self, amount: float = 1) -> None: """ Increment counter (no labels version) - + Args: amount: Increment amount, defaults to 1 """ @@ -91,7 +91,7 @@ def __init__(self, labeled_counter): def inc(self, amount: float = 1) -> None: """ Increment counter - + Args: amount: Increment amount, defaults to 1 """ diff --git a/methods/EverCore/src/core/observation/metrics/gauge.py b/methods/EverCore/src/core/observation/metrics/gauge.py index 863b743d..d8951f18 100644 --- a/methods/EverCore/src/core/observation/metrics/gauge.py +++ b/methods/EverCore/src/core/observation/metrics/gauge.py @@ -16,14 +16,14 @@ class BaseGauge(ABC): """ Gauge base class - + Features: - Can increase or decrease (instantaneous value) - Built-in auto-refresh capability (default 5 seconds) - Must inherit and override refresh() method - Each instance manages its own refresh tasks independently - Supports manual set() method - + Usage - inherit and override refresh method: class KafkaPendingMessagesGauge(BaseGauge): def __init__(self, kafka_consumer): @@ -33,18 +33,18 @@ def __init__(self, kafka_consumer): labelnames=['job_name'] ) self.kafka_consumer = kafka_consumer - + def refresh(self, labels: dict) -> float: '''Return current value''' return len(self.kafka_consumer.pending_messages) - + # Usage 1: Auto-refresh (default 5 seconds) gauge = KafkaPendingMessagesGauge(kafka_consumer) gauge.labels(job_name='tanka').start_refresh() - + # Usage 2: Custom refresh interval gauge.labels(job_name='tanka').start_refresh(interval_seconds=10) - + # Usage 3: Manual set (without auto-refresh) gauge.labels(job_name='tanka').set(42) """ @@ -90,7 +90,7 @@ def __init__( def labels(self, **labels) -> 'LabeledGauge': """ Return a Gauge with labels - + Returns: LabeledGauge instance """ @@ -120,24 +120,24 @@ def dec(self, amount: float = 1) -> None: def refresh(self, labels: dict) -> float: """ Refresh method (subclass must override) - + Args: labels: Label dictionary - + Returns: Current Gauge value - + Notes: - Subclass must override this method to implement custom refresh logic - This method is called periodically by auto-refresh task (default 5 seconds) - Can return any float value, will be automatically updated to Gauge - + Example: class QueueSizeGauge(BaseGauge): def __init__(self, queue): super().__init__('queue_size', 'Queue size') self.queue = queue - + def refresh(self, labels: dict) -> float: return self.queue.qsize() """ @@ -159,7 +159,7 @@ async def _stop_all_refresh_tasks(self) -> None: class LabeledGauge: """ Gauge with labels - + Provides the same interface as native Gauge, with auto-refresh support. """ @@ -198,26 +198,26 @@ def start_refresh( ) -> 'LabeledGauge': """ Start auto-refresh - + Args: interval_seconds: Refresh interval (seconds), default 5 seconds enable_async: Whether to support async refresh method, default True - + Returns: self (supports chaining) - + Example: # Default 5 second refresh gauge.labels(job='tanka').start_refresh() - + # Custom refresh interval gauge.labels(job='tanka').start_refresh(interval_seconds=10) - + # Async refresh method class AsyncGauge(BaseGauge): async def refresh(self, labels: dict) -> float: return await self.get_value_async() - + gauge.labels(type='A').start_refresh(enable_async=True) """ # Stop existing task if any (prevent task leak) @@ -261,7 +261,7 @@ async def stop_refresh(self) -> None: class RefreshTask: """ Refresh task - + Each label combination has an independent refresh task. """ diff --git a/methods/EverCore/src/core/observation/metrics/histogram.py b/methods/EverCore/src/core/observation/metrics/histogram.py index 5742aa73..880b0317 100644 --- a/methods/EverCore/src/core/observation/metrics/histogram.py +++ b/methods/EverCore/src/core/observation/metrics/histogram.py @@ -39,25 +39,25 @@ class HistogramBuckets: class Histogram: """ Histogram metric wrapper - + Features: - Distribution statistics of observed values - Suitable for latency, size, and other distribution data - Automatically calculates percentiles, mean, and sum - + Usage: from core.observation.metrics import Histogram, HistogramBuckets - + request_duration = Histogram( name='http_request_duration_seconds', description='HTTP request duration', labelnames=['method', 'path'], buckets=HistogramBuckets.API_CALL ) - + # Usage request_duration.labels(method='GET', path='/api').observe(0.123) - + # Using context manager with request_duration.labels(method='GET', path='/api').time(): do_something() @@ -101,7 +101,7 @@ def __init__( def labels(self, **labels) -> 'LabeledHistogram': """ Return a Histogram with labels - + Returns: LabeledHistogram instance """ @@ -111,7 +111,7 @@ def labels(self, **labels) -> 'LabeledHistogram': def observe(self, amount: float) -> None: """ Record an observed value (no labels version) - + Args: amount: Observed value """ @@ -120,7 +120,7 @@ def observe(self, amount: float) -> None: def time(self): """ Return a timing context manager (no labels version) - + Usage: with histogram.time(): do_something() @@ -137,7 +137,7 @@ def __init__(self, labeled_histogram): def observe(self, amount: float) -> None: """ Record an observed value - + Args: amount: Observed value """ @@ -146,7 +146,7 @@ def observe(self, amount: float) -> None: def time(self): """ Return a timing context manager - + Usage: with histogram.labels(method='GET').time(): do_something() diff --git a/methods/EverCore/src/core/observation/metrics/registry.py b/methods/EverCore/src/core/observation/metrics/registry.py index 53c4cde4..38f75de7 100644 --- a/methods/EverCore/src/core/observation/metrics/registry.py +++ b/methods/EverCore/src/core/observation/metrics/registry.py @@ -16,10 +16,10 @@ def get_metrics_registry() -> CollectorRegistry: """ Get the global metrics registry - + Returns: CollectorRegistry: Prometheus registry instance - + Notes: - Uses prometheus_client's global REGISTRY by default - All metrics are automatically registered to this registry @@ -34,7 +34,7 @@ def get_metrics_registry() -> CollectorRegistry: def set_metrics_registry(registry: CollectorRegistry) -> None: """ Set custom registry (mainly for testing) - + Args: registry: Custom CollectorRegistry instance """ @@ -45,7 +45,7 @@ def set_metrics_registry(registry: CollectorRegistry) -> None: def generate_metrics_response() -> bytes: """ Generate metrics response content (for testing/debugging) - + Returns: bytes: Prometheus format metrics data """ @@ -55,7 +55,7 @@ def generate_metrics_response() -> bytes: def reset_metrics_registry() -> None: """ Reset metrics registry (mainly for testing) - + Warning: Do not call this method in production """ global _metrics_registry diff --git a/methods/EverCore/src/core/observation/metrics/server.py b/methods/EverCore/src/core/observation/metrics/server.py index 27aad556..588e2779 100644 --- a/methods/EverCore/src/core/observation/metrics/server.py +++ b/methods/EverCore/src/core/observation/metrics/server.py @@ -6,10 +6,10 @@ Usage: from core.observation.metrics.server import start_metrics_server - + # Start metrics server on port 9090 start_metrics_server(port=9090) - + # Or use environment variable METRICS_PORT start_metrics_server() # reads from METRICS_PORT or defaults to 9090 @@ -36,21 +36,21 @@ def start_metrics_server( ) -> bool: """ Start standalone Prometheus metrics HTTP server - + Args: port: Port to listen on (default: from METRICS_PORT env or 9090) addr: Address to bind to (default: 0.0.0.0) - + Returns: bool: True if server started successfully, False if already running - + Example: # Start on default port 9090 start_metrics_server() - + # Start on custom port start_metrics_server(port=9091) - + # Prometheus can scrape: http://your-host:9090/metrics """ global _metrics_server_started @@ -89,11 +89,11 @@ def is_metrics_server_running() -> bool: def get_metrics_url(host: str = "localhost", port: Optional[int] = None) -> str: """ Get the metrics endpoint URL - + Args: host: Hostname (default: localhost) port: Port (default: from METRICS_PORT env or 9090) - + Returns: str: Full URL to metrics endpoint """ diff --git a/methods/EverCore/src/core/oxm/mongo/base_repository.py b/methods/EverCore/src/core/oxm/mongo/base_repository.py index 8f1c9fea..641f47fe 100644 --- a/methods/EverCore/src/core/oxm/mongo/base_repository.py +++ b/methods/EverCore/src/core/oxm/mongo/base_repository.py @@ -242,7 +242,7 @@ async def create_batch( # We need to manually retrieve inserted_ids from the returned InsertManyResult and set them result = await self.model.insert_many(documents, session=session) # Set the _id generated by MongoDB back to the id attribute of each document object - for doc, inserted_id in zip(documents, result.inserted_ids): + for doc, inserted_id in zip(documents, result.inserted_ids, strict=False): doc.id = inserted_id logger.info( "✅ Batch document creation successful [%s]: %d records", diff --git a/methods/EverCore/src/core/oxm/mongo/migration/manager.py b/methods/EverCore/src/core/oxm/mongo/migration/manager.py index b3d61f7e..fd0c6062 100644 --- a/methods/EverCore/src/core/oxm/mongo/migration/manager.py +++ b/methods/EverCore/src/core/oxm/mongo/migration/manager.py @@ -39,39 +39,39 @@ class MigrationManager: class Forward: """Forward migration""" - + # Example: Iterative migration (recommended) # @iterative_migration() # async def update_field(self, input_document: OldModel, output_document: NewModel): # output_document.new_field = input_document.old_field - + # Example: Free fall migration (flexible) # @free_fall_migration(document_models=[YourModel]) # async def create_indexes(self, session): # # Get collection # collection = YourModel.get_pymongo_collection() - # + # # # Create indexes # indexes = [ # IndexModel([("field_name", ASCENDING)], name="idx_field_name") # ] # await collection.create_indexes(indexes) - + pass class Backward: """Backward migration""" - + # @iterative_migration() # async def revert_field(self, input_document: NewModel, output_document: OldModel): # output_document.old_field = input_document.new_field - + # @free_fall_migration(document_models=[YourModel]) # async def drop_indexes(self, session): # collection = YourModel.get_pymongo_collection() # await collection.drop_index("idx_field_name") - + pass ''' @@ -340,8 +340,8 @@ def _log_migration_diff(self, before_names, before_current) -> None: if before_names is None: return - added = sorted(list(after_names - before_names)) - removed = sorted(list(before_names - after_names)) + added = sorted(after_names - before_names) + removed = sorted(before_names - after_names) if added: logger.info("✅ Newly executed scripts: %s", ", ".join(added)) diff --git a/methods/EverCore/src/core/queue/redis_group_queue/redis_group_queue_lua_scripts.py b/methods/EverCore/src/core/queue/redis_group_queue/redis_group_queue_lua_scripts.py index 7dfcbf73..8b908725 100644 --- a/methods/EverCore/src/core/queue/redis_group_queue/redis_group_queue_lua_scripts.py +++ b/methods/EverCore/src/core/queue/redis_group_queue/redis_group_queue_lua_scripts.py @@ -11,34 +11,34 @@ -- Get all active owners local active_owners = redis.call('ZRANGE', owner_zset_key, 0, -1) local owner_count = #active_owners - + if owner_count == 0 then return {0, {}} end - + -- Clean up queue_list for all owners for _, owner_id in ipairs(active_owners) do local queue_list_key = queue_list_prefix .. owner_id redis.call('DEL', queue_list_key) end - + -- Evenly distribute partitions local partitions_per_owner = math.floor(total_partitions / owner_count) local extra_partitions = total_partitions % owner_count - + -- Return assignment results in flat array format for proper conversion by Redis clients local assigned_partitions_flat = {} local partition_index = 1 - + for i, owner_id in ipairs(active_owners) do local queue_list_key = queue_list_prefix .. owner_id local partitions_for_this_owner = partitions_per_owner - + -- First 'extra_partitions' owners get one additional partition if i <= extra_partitions then partitions_for_this_owner = partitions_for_this_owner + 1 end - + local owner_partitions = {} for j = 1, partitions_for_this_owner do local partition_name = string.format("%03d", partition_index) @@ -46,15 +46,15 @@ table.insert(owner_partitions, partition_name) partition_index = partition_index + 1 end - + -- Set expiration time redis.call('EXPIRE', queue_list_key, owner_expire) - + -- Add owner_id and partition list to flat array table.insert(assigned_partitions_flat, owner_id) table.insert(assigned_partitions_flat, owner_partitions) end - + return {owner_count, assigned_partitions_flat} end """ @@ -89,11 +89,11 @@ if added == 1 then -- Update queue expiration time redis.call('EXPIRE', queue_key, queue_expire) - + -- Increment total counter local new_count = redis.call('INCR', counter_key) redis.call('EXPIRE', counter_key, activity_expire) - + return {1, new_count, "Added successfully"} else return {0, current_count, "Message already exists"} @@ -241,11 +241,11 @@ for _, owner_id in ipairs(inactive_owners) do -- Remove from zset redis.call('ZREM', owner_zset_key, owner_id) - + -- Delete corresponding queue_list local queue_list_key = queue_list_prefix .. owner_id redis.call('DEL', queue_list_key) - + cleaned_count = cleaned_count + 1 end @@ -378,13 +378,13 @@ -- Traverse all partitions, attempt to get 1 message from each for _, partition in ipairs(owner_queues) do local queue_key = queue_prefix .. partition - + -- Check if queue has messages local queue_size = redis.call('ZCARD', queue_key) if queue_size > 0 then -- Get score of earliest message local min_result = redis.call('ZRANGE', queue_key, 0, 0, 'WITHSCORES') - + -- Directly compare difference between earliest message and current score if #min_result >= 2 then local earliest_message_score = tonumber(min_result[2]) @@ -476,26 +476,26 @@ for i = 1, total_partitions do local partition_name = string.format("%03d", i) local queue_key = queue_prefix .. partition_name - + -- Get queue size local queue_size = redis.call('ZCARD', queue_key) total_messages_in_queues = total_messages_in_queues + queue_size - + local min_score = 0 local max_score = 0 - + if queue_size > 0 then -- Get minimum and maximum score local min_result = redis.call('ZRANGE', queue_key, 0, 0, 'WITHSCORES') local max_result = redis.call('ZRANGE', queue_key, -1, -1, 'WITHSCORES') - + if #min_result >= 2 then min_score = tonumber(min_result[2]) if global_min_score == nil or min_score < global_min_score then global_min_score = min_score end end - + if #max_result >= 2 then max_score = tonumber(max_result[2]) if global_max_score == nil or max_score > global_max_score then @@ -503,7 +503,7 @@ end end end - + -- Store partition statistics (flat array format) table.insert(partition_stats, partition_name) table.insert(partition_stats, queue_size) diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py index a932feb9..6b9cd96a 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py @@ -347,7 +347,6 @@ def create_new_collection(self) -> TenantAwareCollection: # Use tenant-aware alias name using = self.using origin_alias_name = self._original_alias_name - tenant_aware_alias_name = get_tenant_aware_collection_name(origin_alias_name) new_real_name = generate_new_collection_name(origin_alias_name) tenant_aware_new_real_name = get_tenant_aware_collection_name(new_real_name) diff --git a/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py b/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py index 841b2005..be95979a 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py +++ b/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py @@ -238,9 +238,10 @@ def run( try: # Determine whether to pass the callback function based on whether data migration is needed if migrate_data: - populate_fn = lambda old_col, new_col: migrate_data_callback( - old_col, new_col, batch_size, progress=progress, alias=alias - ) + def populate_fn(old_col, new_col): + return migrate_data_callback( + old_col, new_col, batch_size, progress=progress, alias=alias + ) else: populate_fn = None diff --git a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py index 5e4aa0e3..0f2d505f 100644 --- a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py +++ b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py @@ -1,10 +1,10 @@ """ Sensitive Information Detection Tool - AI-Powered Pre-commit Hook -This tool uses LLM to intelligently scan files for sensitive information +This tool uses LLM to intelligently scan files for sensitive information before committing to GitHub: 1. Credentials: API keys, passwords, secrets, tokens -2. Internal network config: Private IPs, internal domains +2. Internal network config: Private IPs, internal domains 3. Personal data: Real names, phone numbers, emails, ID numbers 4. Test data that may point to real users @@ -182,7 +182,7 @@ class FileCheckResult: ## CRITICAL: Check Configuration Names Carefully For EVERY line that sets a default value for a configuration (e.g., os.getenv("X", "default"), get_env("X", "default")): -- Split the default value by "_" or "-" +- Split the default value by "_" or "-" - Check: Is each word a COMMON English word or standard tech term? - If ANY word is NOT recognizable as a dictionary word, flag it as MEDIUM - This applies to: Kafka topics, queue names, bucket names, service names, group IDs, etc. @@ -197,7 +197,7 @@ class FileCheckResult: ### MEDIUM severity (should review) - Internal IPs/domains that appear to be REAL infrastructure configuration - + Principle: Ask "Does this look like a placeholder/example, or a real server address?" - Placeholder IPs have predictable patterns: x.x.0.1, x.x.1.1, x.x.0.0, x.x.255.255 - Real IPs have arbitrary middle/end segments that look like actual assignments @@ -205,19 +205,19 @@ class FileCheckResult: - Example: 192.168.47.83 looks like a real assigned IP (arbitrary numbers) - Internal configuration names that reveal real infrastructure - + PRINCIPLE: Configuration names should only contain DICTIONARY WORDS or INDUSTRY-STANDARD terms. - + Flag as MEDIUM if a configuration name contains: - Abbreviations that are NOT industry-standard (API, HTTP, DB, SQL, JSON are standard) - Words that are NOT in an English dictionary - Words that look like they could be project names, product names, or company abbreviations - + The test: Can you find this word in a standard English dictionary or official technical documentation? If not, it's likely internal terminology that reveals infrastructure details. - + NOTE: Even in os.getenv("VAR", "fallback") - if the fallback looks specific, flag it! - + - Internal domains that look real (specific hostnames, not generic like "example.internal") - Data that MIGHT be real but you're not sure - Potential real user references that aren't obviously test data @@ -241,7 +241,7 @@ class FileCheckResult: Apply this principle: "Does it look intentionally fake/placeholder, or accidentally real?" - Placeholder patterns: strings with "xxxx", "your-", "${VAR}", "{{...}}", "" -- IPs that are OBVIOUSLY examples: +- IPs that are OBVIOUSLY examples: Pattern: ends with .0.1, .1.1, .0.0, .255.255, or is localhost/127.0.0.1 These are universally recognized defaults, not real infrastructure - Generic/example domains: contains "example", "test", "demo", "sample", "foo", "bar", or is localhost diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py index c5ae9f72..ebf37b22 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py @@ -46,7 +46,6 @@ def from_mongo(cls, source_doc: AgentSkillRecord) -> Dict[str, Any]: try: name = source_doc.name or "" description = source_doc.description or "" - content = source_doc.content or "" # Primary text field: name + description combined content_field = "\n".join(s for s in [name, description] if s) diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py index ea5299b2..cff51438 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py @@ -49,7 +49,6 @@ def from_mongo(cls, source_doc: MongoUserProfile) -> List[Dict[str, Any]]: try: profile_data: Dict[str, Any] = source_doc.profile_data or {} - doc_id = str(source_doc.id) if source_doc.id else "" user_id = source_doc.user_id or "" group_id = source_doc.group_id or "" scenario = source_doc.scenario or ScenarioType.SOLO.value diff --git a/methods/EverCore/src/manage.py b/methods/EverCore/src/manage.py index a0345cad..eac4c004 100644 --- a/methods/EverCore/src/manage.py +++ b/methods/EverCore/src/manage.py @@ -160,13 +160,13 @@ def shell( banner = """ ======================================== Memsys Backend Shell - + Available variables: - app: FastAPI application instance - app_state: Application state (if available) - graphs: LangGraph instances (if available) - logger: Logger instance - + Example usage: >>> logger.info("Hello from shell!") >>> app.routes # View all routes diff --git a/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py b/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py index f2d40f4f..56b91b29 100644 --- a/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py +++ b/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py @@ -8,7 +8,9 @@ from typing import Dict, Any, Optional, List from datetime import datetime from dataclasses import dataclass, field -import json, re, os +import json +import re +import os from core.di.utils import get_bean_by_type from core.component.llm.tokenizer.tokenizer_factory import TokenizerFactory from common_utils.datetime_utils import from_iso_format as dt_from_iso_format diff --git a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py index 3bd148a2..e2139071 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py @@ -8,7 +8,8 @@ from dataclasses import dataclass from typing import Optional, List, Dict, Any from datetime import datetime -import re, json +import re +import json from memory_layer.prompts import get_prompt_by @@ -224,7 +225,6 @@ async def _extract_episode( prompt_template = self.episode_generation_prompt content_key = "conversation" time_key = "conversation_start_time" - default_title = "Conversation Episode" else: return None diff --git a/methods/EverCore/src/memory_layer/memory_manager.py b/methods/EverCore/src/memory_layer/memory_manager.py index cbebf481..01a26316 100644 --- a/methods/EverCore/src/memory_layer/memory_manager.py +++ b/methods/EverCore/src/memory_layer/memory_manager.py @@ -296,7 +296,7 @@ async def extract_memory( return result - except Exception as e: + except Exception: status = 'error' raise finally: @@ -375,12 +375,12 @@ async def _extract_foresight( if uid is None: display_name = ",".join( - set( - [ + { + item.get("message", item).get("sender_name") for item in memcell.original_data or [] - ] - ) + + } ) else: for item in memcell.original_data or []: diff --git a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py index 075d73d0..ebc8948e 100644 --- a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py +++ b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py @@ -184,7 +184,7 @@ async def index_profile( # Step 4: Add vectors to entities valid_entities = [] - for entity, vector in zip(entities, vectors): + for entity, vector in zip(entities, vectors, strict=False): if vector is not None and len(vector) > 0: entity["vector"] = vector valid_entities.append(entity) diff --git a/methods/EverCore/src/service/memcell_delete_service.py b/methods/EverCore/src/service/memcell_delete_service.py index 871d5c5d..c3b4e02d 100644 --- a/methods/EverCore/src/service/memcell_delete_service.py +++ b/methods/EverCore/src/service/memcell_delete_service.py @@ -203,7 +203,7 @@ async def _gather_deletes(self, *tasks: tuple[str, Any, dict]) -> dict[str, int] reraise_critical_errors(results) counts: dict[str, int] = {} - for name, result in zip(names, results): + for name, result in zip(names, results, strict=False): if isinstance(result, Exception): logger.error("Failed to cascade delete %s: %s", name, result) else: @@ -232,13 +232,13 @@ async def _batch_delete_records( if user_id == MAGIC_ALL and group_id == MAGIC_ALL: return {"episodes": 0, "atomic_facts": 0, "foresights": 0} - mongo_kwargs = dict( - user_id=user_id, - group_id=group_id, - session_id=session_id, - sender_id=sender_id, - ) - scope_kwargs = dict(user_id=user_id, group_id=group_id) + mongo_kwargs = { + "user_id": user_id, + "group_id": group_id, + "session_id": session_id, + "sender_id": sender_id, + } + scope_kwargs = {"user_id": user_id, "group_id": group_id} # RawMessage and MemCell are source data — not deleted by filters. # Milvus/ES only support user_id/group_id (no session_id/sender_id). From 20b54bfc8b727a4f2c927ead66024299bb812fe6 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 19:14:44 +0800 Subject: [PATCH 09/24] Restore F841 deletions and ignore the rule going forward MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous --unsafe-fixes commit (dabc3c4) auto-removed assignments that ruff classified as unused locals. Several of those were not safe to drop on a purely-syntactic read: - agentic_layer/memory_manager.py: three `memory_type = (...)` bindings in `retrieve_mem_keyword / _vector / _hybrid`. These were paired with `start_time = time.perf_counter()` in the original metrics-labelling commit (b3958cb). The paired start_time was removed during a later metrics refactor that pushed recording into the inner `get_*_search_results` helpers; the memory_type binding got orphaned but the value is still a useful debugging breadcrumb at the wrapper layer. - core/asynctasks/task_manager.py: `job = await pool.enqueue_job(...)` — keeping the name binding aids debugging and any future follow-up that needs the job handle. - memory_manager.py `extend_data = fields['extend_data']`, agent_skill_milvus_converter `content = source_doc.content or ""`, episode_memory_extractor `default_title = "Conversation Episode"`, test_di_scanner `initial_bean_count = len(...)`, tenant_aware_collection_with_suffix `tenant_aware_alias_name = ...`, user_profile_milvus_converter `doc_id = str(source_doc.id) ...`: same story — likely vestiges of in-flight work or intentional reads kept for clarity. Restoring them is cheap; auto-removing was the wrong default. Add F841 to the ruff `ignore` list in pyproject.toml so future `--unsafe-fixes` runs won't reintroduce the same destructive cleanup. F841 can still be surfaced explicitly via `ruff check --select F841` when a human wants to triage them; we just don't auto-fix. --- methods/EverCore/pyproject.toml | 6 ++++++ methods/EverCore/src/agentic_layer/memory_manager.py | 8 ++++---- methods/EverCore/src/core/asynctasks/task_manager.py | 2 +- methods/EverCore/src/core/di/tests/test_di_scanner.py | 1 + .../oxm/milvus/tenant_aware_collection_with_suffix.py | 1 + .../milvus/converter/agent_skill_milvus_converter.py | 1 + .../milvus/converter/user_profile_milvus_converter.py | 1 + .../memory_extractor/episode_memory_extractor.py | 1 + 8 files changed, 16 insertions(+), 5 deletions(-) diff --git a/methods/EverCore/pyproject.toml b/methods/EverCore/pyproject.toml index 8862493d..e9abf0ca 100644 --- a/methods/EverCore/pyproject.toml +++ b/methods/EverCore/pyproject.toml @@ -137,6 +137,12 @@ ignore = [ "E501", # line too long — formatter's job "B008", # function calls in argument defaults (FastAPI uses Depends()) "B904", # raise from inside except (large existing baseline) + # F841 (unused-variable) intentionally not enforced: `--unsafe-fixes` + # would auto-delete assignments. Some "unused" locals are kept on purpose + # — e.g. names that aid debugging, vestigial metric labels paired with + # other state, and `var = await ...` bindings where the await is the + # real intent. Manual review is required; auto-removal is unsafe. + "F841", ] [tool.ruff.lint.per-file-ignores] diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index 5ddb0296..082ef3ea 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -447,7 +447,7 @@ async def retrieve_mem_keyword( """Keyword-based memory retrieval""" top_k = retrieve_mem_request.top_k is_unlimited_mode = top_k == -1 - ( + memory_type = ( retrieve_mem_request.memory_types[0].value if retrieve_mem_request.memory_types else 'unknown' @@ -577,7 +577,7 @@ async def retrieve_mem_vector( """Vector-based memory retrieval""" top_k = retrieve_mem_request.top_k is_unlimited_mode = top_k == -1 - ( + memory_type = ( retrieve_mem_request.memory_types[0].value if retrieve_mem_request.memory_types else 'unknown' @@ -791,7 +791,7 @@ async def retrieve_mem_hybrid( self, retrieve_mem_request: 'RetrieveMemRequest' ) -> RetrieveMemResponse: """Hybrid memory retrieval: keyword + vector + rerank""" - ( + memory_type = ( retrieve_mem_request.memory_types[0].value if retrieve_mem_request.memory_types else 'unknown' @@ -1305,7 +1305,7 @@ async def group_by_groupid_stratagy( atomic_fact = fields['atomic_fact'] foresight = fields['foresight'] evidence = fields['evidence'] - fields['extend_data'] + extend_data = fields['extend_data'] search_source = fields['search_source'] # Process timestamp timestamp = from_iso_format(timestamp_raw) diff --git a/methods/EverCore/src/core/asynctasks/task_manager.py b/methods/EverCore/src/core/asynctasks/task_manager.py index 6005ac73..4626f18b 100644 --- a/methods/EverCore/src/core/asynctasks/task_manager.py +++ b/methods/EverCore/src/core/asynctasks/task_manager.py @@ -331,7 +331,7 @@ async def enqueue_task( defer_until = get_now_with_timezone() + timedelta(seconds=delay) # Enqueue task - await pool.enqueue_job( + job = await pool.enqueue_job( task_name, task_context, *args, diff --git a/methods/EverCore/src/core/di/tests/test_di_scanner.py b/methods/EverCore/src/core/di/tests/test_di_scanner.py index b9d9447f..90a4ae45 100644 --- a/methods/EverCore/src/core/di/tests/test_di_scanner.py +++ b/methods/EverCore/src/core/di/tests/test_di_scanner.py @@ -281,6 +281,7 @@ class TestConditionalRegistration: def test_lazy_registration(self): """Test lazy registration""" container = get_container() + initial_bean_count = len(container._named_beans) # Define component with lazy registration @component(name="lazy_comp_unique_1", lazy=True) diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py index 6b9cd96a..a932feb9 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py @@ -347,6 +347,7 @@ def create_new_collection(self) -> TenantAwareCollection: # Use tenant-aware alias name using = self.using origin_alias_name = self._original_alias_name + tenant_aware_alias_name = get_tenant_aware_collection_name(origin_alias_name) new_real_name = generate_new_collection_name(origin_alias_name) tenant_aware_new_real_name = get_tenant_aware_collection_name(new_real_name) diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py index ebf37b22..c5ae9f72 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/agent_skill_milvus_converter.py @@ -46,6 +46,7 @@ def from_mongo(cls, source_doc: AgentSkillRecord) -> Dict[str, Any]: try: name = source_doc.name or "" description = source_doc.description or "" + content = source_doc.content or "" # Primary text field: name + description combined content_field = "\n".join(s for s in [name, description] if s) diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py index cff51438..ea5299b2 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py @@ -49,6 +49,7 @@ def from_mongo(cls, source_doc: MongoUserProfile) -> List[Dict[str, Any]]: try: profile_data: Dict[str, Any] = source_doc.profile_data or {} + doc_id = str(source_doc.id) if source_doc.id else "" user_id = source_doc.user_id or "" group_id = source_doc.group_id or "" scenario = source_doc.scenario or ScenarioType.SOLO.value diff --git a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py index e2139071..4b6985bd 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py @@ -225,6 +225,7 @@ async def _extract_episode( prompt_template = self.episode_generation_prompt content_key = "conversation" time_key = "conversation_start_time" + default_title = "Conversation Episode" else: return None From 783c84204b6e573fe61a0a689f89b1810b61d502 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 19:23:10 +0800 Subject: [PATCH 10/24] Finish EverCore quality gate setup --- methods/EverCore/Makefile | 4 +- methods/EverCore/pyproject.toml | 13 ++- .../src/agentic_layer/agentic_utils.py | 4 +- .../src/agentic_layer/filter_parser.py | 2 +- .../src/agentic_layer/memory_manager.py | 6 +- .../src/agentic_layer/metrics/__init__.py | 6 -- .../agentic_layer/metrics/rerank_metrics.py | 35 ++----- .../metrics/vectorize_metrics.py | 36 +++---- .../agentic_layer/profile_search_service.py | 28 +++--- .../src/agentic_layer/retrieval_utils.py | 10 +- .../src/agentic_layer/search_mem_service.py | 3 +- .../src/agentic_layer/vectorize_interface.py | 12 ++- .../src/agentic_layer/vectorize_vllm.py | 1 - methods/EverCore/src/api_specs/dtos/memory.py | 15 +-- .../EverCore/src/api_specs/memory_types.py | 4 +- .../src/api_specs/request_converter.py | 12 +-- .../EverCore/src/biz_layer/mem_memorize.py | 20 ++-- .../EverCore/src/biz_layer/memorize_config.py | 7 +- .../src/common_utils/language_utils.py | 1 - .../redis_cache_queue/redis_data_processor.py | 6 +- .../llm/llm_adapter/gemini_adapter.py | 11 +-- .../llm/tokenizer/tokenizer_factory.py | 14 ++- methods/EverCore/src/core/di/container.py | 4 +- .../src/core/di/tests/test_di_scanner.py | 5 +- .../interface/controller/base_controller.py | 18 ++-- .../src/core/lifespan/database_lifespan.py | 7 +- .../src/core/lifespan/metrics_lifespan.py | 2 +- .../src/core/lifespan/mongodb_lifespan.py | 6 +- .../src/core/lock/redis_distributed_lock.py | 4 +- .../middleware/database_session_middleware.py | 5 +- .../core/middleware/prometheus_middleware.py | 30 +++--- .../middleware/sse_exception_middleware.py | 2 +- .../EverCore/src/core/observation/logger.py | 2 +- .../src/core/observation/metrics/__init__.py | 11 +-- .../src/core/observation/metrics/counter.py | 2 +- .../src/core/observation/metrics/gauge.py | 16 ++- .../src/core/observation/metrics/histogram.py | 52 +++++++++- .../src/core/observation/metrics/registry.py | 2 +- .../src/core/observation/metrics/server.py | 13 +-- methods/EverCore/src/core/oxm/constants.py | 6 +- methods/EverCore/src/core/oxm/es/analyzer.py | 4 +- .../src/core/oxm/milvus/migration/utils.py | 12 +-- .../EverCore/src/core/oxm/mongo/__init__.py | 6 +- .../src/core/oxm/mongo/document_base.py | 1 - .../EverCore/src/core/oxm/pg/audit_base.py | 8 +- .../msg_group_queue_manager.py | 28 ++++-- methods/EverCore/src/core/request/__init__.py | 1 - .../src/core/request/timeout_background.py | 2 +- .../src/core/tenants/tenant_config.py | 4 +- .../tenantize/kv/redis/tenant_key_utils.py | 2 - .../oxm/es/tenant_field_es_interceptor.py | 11 +-- .../oxm/mongo/tenant_aware_mongo_client.py | 4 +- .../mongo/tenant_field_command_interceptor.py | 3 +- .../data_fix/milvus_rebuild_collection.py | 1 + .../src/devops_scripts/i18n/i18n_tool.py | 14 +-- .../milvus_admin/browse_collections.py | 4 +- .../adapters/input/api/dto/memory_dto.py | 1 + .../repository/mem_scene_raw_repository.py | 5 +- .../repository/memcell_raw_repository.py | 4 +- .../repository/user_profile_raw_repository.py | 10 +- .../user_profile_milvus_converter.py | 12 ++- .../repository/atomic_fact_es_repository.py | 12 ++- .../episodic_memory_es_repository.py | 12 ++- .../repository/foresight_es_repository.py | 12 ++- methods/EverCore/src/manage.py | 4 +- .../memory_layer/cluster_manager/manager.py | 67 +++++-------- .../EverCore/src/memory_layer/llm/__init__.py | 10 +- .../EverCore/src/memory_layer/llm/config.py | 4 +- .../agent_memcell_extractor.py | 16 ++- .../conv_memcell_extractor.py | 4 +- .../memory_extractor/agent_case_extractor.py | 35 ++++--- .../memory_extractor/agent_skill_extractor.py | 54 ++++++----- .../memory_extractor/atomic_fact_extractor.py | 4 +- .../episode_memory_extractor.py | 2 +- .../memory_extractor/foresight_extractor.py | 2 +- .../memory_extractor/profile_extractor.py | 78 +++++++++++---- .../src/memory_layer/memory_manager.py | 11 +-- .../memory_layer/profile_indexer/__init__.py | 5 +- .../profile_indexer/profile_indexer.py | 4 +- .../memory_layer/profile_manager/manager.py | 3 +- .../src/memory_layer/prompts/__init__.py | 7 +- .../prompts/en/atomic_fact_prompts.py | 2 +- .../memory_layer/prompts/zh/conv_prompts.py | 2 +- .../prompts/zh/group_profile_merge_prompts.py | 1 - methods/EverCore/src/run.py | 1 + .../src/service/memcell_delete_service.py | 3 +- .../EverCore/src/service/settings_service.py | 5 +- methods/EverCore/uv.lock | 97 ++++++++++++++++++- 88 files changed, 567 insertions(+), 464 deletions(-) diff --git a/methods/EverCore/Makefile b/methods/EverCore/Makefile index e1bba591..a45379e3 100644 --- a/methods/EverCore/Makefile +++ b/methods/EverCore/Makefile @@ -58,7 +58,7 @@ ruff-fix: # wording may shift between minor releases. typecheck: @echo "Running ty..." - uv run ty check + uv run ty check --ignore all --error possibly-unresolved-reference # Run pyright type checker (fallback). # pyrightconfig.json controls scope and strictness; current mode is "off" @@ -69,7 +69,7 @@ typecheck-pyright: # Run tests test: - PYTHONPATH=src pytest tests/ + PYTHONPATH=src uv run pytest tests/ # Clean up clean: diff --git a/methods/EverCore/pyproject.toml b/methods/EverCore/pyproject.toml index e9abf0ca..283545c7 100644 --- a/methods/EverCore/pyproject.toml +++ b/methods/EverCore/pyproject.toml @@ -135,8 +135,15 @@ select = [ ] ignore = [ "E501", # line too long — formatter's job + "E402", # module-level imports after setup/cycle guards need manual review + "E741", # single-letter glyph fields exist in current CLI box styling "B008", # function calls in argument defaults (FastAPI uses Depends()) + "B019", # cached methods require lifetime review before changing behavior + "B024", # ABC marker classes exist in the current DI/controller baseline + "B026", # logger wrappers intentionally pass exc_info before *args "B904", # raise from inside except (large existing baseline) + "ASYNC109", # timeout parameters are part of several public wrapper APIs + "ASYNC240", # pathlib calls in async code need targeted async-IO migration # F841 (unused-variable) intentionally not enforced: `--unsafe-fixes` # would auto-delete assignments. Some "unused" locals are kept on purpose # — e.g. names that aid debugging, vestigial metric labels paired with @@ -166,8 +173,9 @@ root = ["src"] include = ["src"] [tool.ty.rules] -# Catches "variable assigned inside try, referenced inside except" patterns -# (e.g., the milvus_start / user_id_list bugs surfaced in the audit). +# Catches "variable assigned inside try, referenced inside except" patterns. +# `make typecheck` passes `--ignore all --error possibly-unresolved-reference` +# so this first ty rollout stays focused instead of failing on the full baseline. possibly-unresolved-reference = "error" [tool.pytest.ini_options] @@ -190,6 +198,7 @@ dev = [ "pytest>=8.4.2", "pytest-asyncio>=1.1.0", "pytest-cov>=6.0.0", + "psutil>=7.0.0", "nest-asyncio", "typer", "pyinstrument", diff --git a/methods/EverCore/src/agentic_layer/agentic_utils.py b/methods/EverCore/src/agentic_layer/agentic_utils.py index d895e896..1c7322d3 100644 --- a/methods/EverCore/src/agentic_layer/agentic_utils.py +++ b/methods/EverCore/src/agentic_layer/agentic_utils.py @@ -89,7 +89,9 @@ class AgenticConfig: round1_emb_top_n: int = 50 # Number of embedding candidates round1_bm25_top_n: int = 50 # Number of BM25 candidates round1_top_n: int = 20 # Number returned after RRF fusion - round1_rerank_top_n: int = int(os.getenv("AGENTIC_ROUND1_RERANK_TOP_N", "10")) # Number after reranking used for LLM judgment + round1_rerank_top_n: int = int( + os.getenv("AGENTIC_ROUND1_RERANK_TOP_N", "10") + ) # Number after reranking used for LLM judgment # LLM configuration llm_temperature: float = 0.0 # Low temperature for judgment diff --git a/methods/EverCore/src/agentic_layer/filter_parser.py b/methods/EverCore/src/agentic_layer/filter_parser.py index c4b860ff..0c3d61f8 100644 --- a/methods/EverCore/src/agentic_layer/filter_parser.py +++ b/methods/EverCore/src/agentic_layer/filter_parser.py @@ -142,7 +142,7 @@ def parse( def parse_mongo_filters( - filters: Dict[str, Any] + filters: Dict[str, Any], ) -> Tuple[Dict[str, Any], Optional[str], Optional[List[str]]]: """Convenience function wrapping MongoFilterParser.parse().""" return MongoFilterParser.parse(filters) diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index 082ef3ea..1118c0fb 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -54,9 +54,7 @@ from core.observation.tracing.decorators import trace_logger from core.observation.stage_timer import timed, timed_parallel from core.nlp.stopwords_utils import filter_stopwords -from common_utils.datetime_utils import ( - from_iso_format, -) +from common_utils.datetime_utils import from_iso_format from infra_layer.adapters.out.persistence.repository.memcell_raw_repository import ( MemCellRawRepository, ) @@ -1355,7 +1353,7 @@ async def group_by_groupid_stratagy( task_intent=fields.get('task_intent', ''), approach=fields.get('approach', ''), quality_score=fields.get('quality_score'), - key_insight=fields.get('key_insight', '') + key_insight=fields.get('key_insight', ''), ) case MemoryType.AGENT_SKILL.value: # AgentSkill doesn't have parent_type/parent_id fields diff --git a/methods/EverCore/src/agentic_layer/metrics/__init__.py b/methods/EverCore/src/agentic_layer/metrics/__init__.py index b419ae93..89dd8f37 100644 --- a/methods/EverCore/src/agentic_layer/metrics/__init__.py +++ b/methods/EverCore/src/agentic_layer/metrics/__init__.py @@ -57,35 +57,29 @@ 'VECTORIZE_FALLBACK_TOTAL', 'VECTORIZE_ERRORS_TOTAL', 'VECTORIZE_TOKENS_TOTAL', - # Rerank metrics 'RERANK_REQUESTS_TOTAL', 'RERANK_DURATION_SECONDS', 'RERANK_DOCUMENTS_TOTAL', 'RERANK_FALLBACK_TOTAL', 'RERANK_ERRORS_TOTAL', - # Retrieve metrics 'RETRIEVE_REQUESTS_TOTAL', 'RETRIEVE_DURATION_SECONDS', 'RETRIEVE_RESULTS_COUNT', 'RETRIEVE_STAGE_DURATION_SECONDS', 'RETRIEVE_ERRORS_TOTAL', - # Memorize metrics 'MEMORIZE_REQUESTS_TOTAL', 'MEMORIZE_DURATION_SECONDS', 'MEMORIZE_MESSAGES_TOTAL', 'MEMORIZE_ERRORS_TOTAL', - # Boundary detection metrics 'BOUNDARY_DETECTION_TOTAL', 'MEMCELL_EXTRACTED_TOTAL', - # Memory extraction metrics 'MEMORY_EXTRACTION_STAGE_DURATION_SECONDS', 'MEMORY_EXTRACTED_TOTAL', 'EXTRACT_MEMORY_REQUESTS_TOTAL', 'EXTRACT_MEMORY_DURATION_SECONDS', ] - diff --git a/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py b/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py index 49f03603..61675f44 100644 --- a/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/rerank_metrics.py @@ -129,11 +129,9 @@ # Helper Functions # ============================================================ + def record_rerank_request( - provider: str, - status: str, - duration_seconds: float, - documents_count: int, + provider: str, status: str, duration_seconds: float, documents_count: int ) -> None: """ Helper function to record all rerank metrics in one call @@ -153,26 +151,17 @@ def record_rerank_request( ) """ # Counter - RERANK_REQUESTS_TOTAL.labels( - provider=provider, - status=status - ).inc() + RERANK_REQUESTS_TOTAL.labels(provider=provider, status=status).inc() # Duration histogram - RERANK_DURATION_SECONDS.labels( - provider=provider - ).observe(duration_seconds) + RERANK_DURATION_SECONDS.labels(provider=provider).observe(duration_seconds) # Documents count histogram - RERANK_DOCUMENTS_TOTAL.labels( - provider=provider - ).observe(documents_count) + RERANK_DOCUMENTS_TOTAL.labels(provider=provider).observe(documents_count) def record_rerank_fallback( - primary_provider: str, - fallback_provider: str, - reason: str, + primary_provider: str, fallback_provider: str, reason: str ) -> None: """ Helper function to record rerank fallback event @@ -192,14 +181,11 @@ def record_rerank_fallback( RERANK_FALLBACK_TOTAL.labels( primary_provider=primary_provider, fallback_provider=fallback_provider, - reason=reason + reason=reason, ).inc() -def record_rerank_error( - provider: str, - error_type: str, -) -> None: +def record_rerank_error(provider: str, error_type: str) -> None: """ Helper function to record rerank error @@ -213,7 +199,4 @@ def record_rerank_error( error_type='timeout' ) """ - RERANK_ERRORS_TOTAL.labels( - provider=provider, - error_type=error_type - ).inc() + RERANK_ERRORS_TOTAL.labels(provider=provider, error_type=error_type).inc() diff --git a/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py b/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py index 2ea610a6..dd952098 100644 --- a/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/vectorize_metrics.py @@ -124,6 +124,7 @@ # Helper Functions # ============================================================ + def record_vectorize_request( provider: str, operation: str, @@ -155,23 +156,19 @@ def record_vectorize_request( """ # Counter VECTORIZE_REQUESTS_TOTAL.labels( - provider=provider, - operation=operation, - status=status + provider=provider, operation=operation, status=status ).inc() # Duration histogram - VECTORIZE_DURATION_SECONDS.labels( - provider=provider, - operation=operation - ).observe(duration_seconds) + VECTORIZE_DURATION_SECONDS.labels(provider=provider, operation=operation).observe( + duration_seconds + ) # Batch size histogram (only for batch operations) if batch_size > 1: - VECTORIZE_BATCH_SIZE.labels( - provider=provider, - operation=operation - ).observe(batch_size) + VECTORIZE_BATCH_SIZE.labels(provider=provider, operation=operation).observe( + batch_size + ) # Token counter (if available) if tokens > 0: @@ -179,9 +176,7 @@ def record_vectorize_request( def record_vectorize_fallback( - primary_provider: str, - fallback_provider: str, - reason: str, + primary_provider: str, fallback_provider: str, reason: str ) -> None: """ Helper function to record vectorize fallback event @@ -201,15 +196,11 @@ def record_vectorize_fallback( VECTORIZE_FALLBACK_TOTAL.labels( primary_provider=primary_provider, fallback_provider=fallback_provider, - reason=reason + reason=reason, ).inc() -def record_vectorize_error( - provider: str, - operation: str, - error_type: str, -) -> None: +def record_vectorize_error(provider: str, operation: str, error_type: str) -> None: """ Helper function to record vectorize error @@ -226,8 +217,5 @@ def record_vectorize_error( ) """ VECTORIZE_ERRORS_TOTAL.labels( - provider=provider, - operation=operation, - error_type=error_type + provider=provider, operation=operation, error_type=error_type ).inc() - diff --git a/methods/EverCore/src/agentic_layer/profile_search_service.py b/methods/EverCore/src/agentic_layer/profile_search_service.py index 5ba7a1f0..80efe058 100644 --- a/methods/EverCore/src/agentic_layer/profile_search_service.py +++ b/methods/EverCore/src/agentic_layer/profile_search_service.py @@ -46,7 +46,11 @@ def parse_embed_text(embed_text: str, item_type: str) -> Dict[str, str]: result = {} if not embed_text: - return {"category": "", "description": ""} if item_type == "explicit_info" else {"trait_name": "", "description": ""} + return ( + {"category": "", "description": ""} + if item_type == "explicit_info" + else {"trait_name": "", "description": ""} + ) # Split by first colon parts = embed_text.split(":", 1) @@ -84,10 +88,7 @@ class ProfileSearchService: No reranking step - directly returns Milvus results with score threshold. """ - def __init__( - self, - milvus_repo: Optional[UserProfileMilvusRepository] = None, - ): + def __init__(self, milvus_repo: Optional[UserProfileMilvusRepository] = None): """Initialize service Args: @@ -127,13 +128,7 @@ async def search_profiles( """ start_time = time.perf_counter() - result = { - "profiles": [], - "metadata": { - "profile_count": 0, - "latency_ms": 0, - } - } + result = {"profiles": [], "metadata": {"profile_count": 0, "latency_ms": 0}} if not query: logger.warning("Empty query for profile search") @@ -203,9 +198,7 @@ async def search_profiles( # Log profile scores for debugging if profiles: scores_str = ", ".join([f"{p['score']:.4f}" for p in profiles]) - logger.info( - f"📊 Profile scores: [{scores_str}]" - ) + logger.info(f"📊 Profile scores: [{scores_str}]") logger.info( "Profile search completed: user_id=%s, group_id=%s, " @@ -227,11 +220,12 @@ async def search_profiles( e, exc_info=True, ) - result["metadata"]["latency_ms"] = int((time.perf_counter() - start_time) * 1000) + result["metadata"]["latency_ms"] = int( + (time.perf_counter() - start_time) * 1000 + ) return result def get_profile_search_service() -> ProfileSearchService: """Get ProfileSearchService from DI container""" return get_bean_by_type(ProfileSearchService) - diff --git a/methods/EverCore/src/agentic_layer/retrieval_utils.py b/methods/EverCore/src/agentic_layer/retrieval_utils.py index e1667557..2f2a6282 100644 --- a/methods/EverCore/src/agentic_layer/retrieval_utils.py +++ b/methods/EverCore/src/agentic_layer/retrieval_utils.py @@ -106,7 +106,9 @@ async def search_with_bm25( scores = bm25.get_scores(tokenized_query) # Sort and return Top-K - results = sorted(zip(candidates, scores, strict=False), key=lambda x: x[1], reverse=True)[:top_k] + results = sorted( + zip(candidates, scores, strict=False), key=lambda x: x[1], reverse=True + )[:top_k] return results @@ -612,9 +614,9 @@ async def agentic_retrieval( "total_latency_ms": 0.0, } - logger.info(f"{'='*60}") + logger.info(f"{'=' * 60}") logger.info(f"Agentic Retrieval: {query[:60]}...") - logger.info(f"{'='*60}") + logger.info(f"{'=' * 60}") # ========== Round 1: Hybrid search Top 20 ========== logger.info("Round 1: Hybrid search for Top 20...") @@ -822,6 +824,6 @@ async def agentic_retrieval( logger.info( f"Complete: Final {len(final_results)} docs | Latency {metadata['total_latency_ms']:.0f}ms" ) - logger.info(f"{'='*60}\n") + logger.info(f"{'=' * 60}\n") return final_results, metadata diff --git a/methods/EverCore/src/agentic_layer/search_mem_service.py b/methods/EverCore/src/agentic_layer/search_mem_service.py index 92f3966e..c73f3764 100644 --- a/methods/EverCore/src/agentic_layer/search_mem_service.py +++ b/methods/EverCore/src/agentic_layer/search_mem_service.py @@ -554,7 +554,6 @@ async def search_memories( original_data = await self._fetch_original_data(episodes, profiles) with timed("assemble_results"): - # Apply top_k limit if top_k > 0: episodes = episodes[:top_k] @@ -1522,7 +1521,7 @@ async def _search_agent_skills( query=query, hits=merged_hits, top_k=rerank_top_k, - instruction="Determine whether the skill's methodology and domain are applicable to the query, preferring same-domain skills with directly relevant steps." + instruction="Determine whether the skill's methodology and domain are applicable to the query, preferring same-domain skills with directly relevant steps.", ) rerank_ms = (time.perf_counter() - stage_start) * 1000 record_retrieve_stage( diff --git a/methods/EverCore/src/agentic_layer/vectorize_interface.py b/methods/EverCore/src/agentic_layer/vectorize_interface.py index ca2d909b..a8457717 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_interface.py +++ b/methods/EverCore/src/agentic_layer/vectorize_interface.py @@ -41,14 +41,20 @@ async def get_embedding_with_usage( @abstractmethod async def get_embeddings( - self, texts: List[str], instruction: Optional[str] = None, is_query: bool = False + self, + texts: List[str], + instruction: Optional[str] = None, + is_query: bool = False, ) -> List[np.ndarray]: """Get embeddings for multiple texts""" pass @abstractmethod async def get_embeddings_batch( - self, text_batches: List[List[str]], instruction: Optional[str] = None, is_query: bool = False + self, + text_batches: List[List[str]], + instruction: Optional[str] = None, + is_query: bool = False, ) -> List[List[np.ndarray]]: """Get embeddings for multiple batches""" pass @@ -66,5 +72,5 @@ async def close(self): class VectorizeError(Exception): """Vectorize API error exception class""" - pass + pass diff --git a/methods/EverCore/src/agentic_layer/vectorize_vllm.py b/methods/EverCore/src/agentic_layer/vectorize_vllm.py index 9ed79a59..c72be875 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_vllm.py +++ b/methods/EverCore/src/agentic_layer/vectorize_vllm.py @@ -54,4 +54,3 @@ def _should_pass_dimensions(self) -> bool: def _should_truncate_client_side(self) -> bool: """vLLM services need client-side truncation""" return True - diff --git a/methods/EverCore/src/api_specs/dtos/memory.py b/methods/EverCore/src/api_specs/dtos/memory.py index d4a300a8..cb1c7e9a 100644 --- a/methods/EverCore/src/api_specs/dtos/memory.py +++ b/methods/EverCore/src/api_specs/dtos/memory.py @@ -19,22 +19,11 @@ from api_specs.memory_types import ScenarioType from bson import ObjectId -from pydantic import ( - BaseModel, - Field, - field_validator, - model_validator, - SkipValidation, -) +from pydantic import BaseModel, Field, field_validator, model_validator, SkipValidation from api_specs.dtos.base import BaseApiResponse from api_specs.memory_types import RetrieveMemoryModel, RawDataType -from api_specs.memory_models import ( - MemoryType, - Metadata, - QueryMetadata, - RetrieveMethod, -) +from api_specs.memory_models import MemoryType, Metadata, QueryMetadata, RetrieveMethod from core.oxm.constants import MAGIC_ALL, MAX_RETRIEVE_LIMIT from biz_layer.retrieve_constants import MAX_GROUP_IDS_COUNT diff --git a/methods/EverCore/src/api_specs/memory_types.py b/methods/EverCore/src/api_specs/memory_types.py index 9ce9b004..21a082dc 100644 --- a/methods/EverCore/src/api_specs/memory_types.py +++ b/methods/EverCore/src/api_specs/memory_types.py @@ -546,7 +546,9 @@ def to_readable_profile(self) -> str: lines.append("") lines.append("[Implicit Traits]") for trait in self.implicit_traits: - lines.append(f" - {trait.get('trait', '')}: {trait.get('description', '')}") + lines.append( + f" - {trait.get('trait', '')}: {trait.get('description', '')}" + ) return "\n".join(lines) if lines else "No profile data yet." diff --git a/methods/EverCore/src/api_specs/request_converter.py b/methods/EverCore/src/api_specs/request_converter.py index 1250e181..127b1ff7 100644 --- a/methods/EverCore/src/api_specs/request_converter.py +++ b/methods/EverCore/src/api_specs/request_converter.py @@ -232,7 +232,7 @@ def _unix_ms_to_datetime(unix_ms: int) -> datetime: def convert_personal_add_to_memorize_request( - request_data: Dict[str, Any] + request_data: Dict[str, Any], ) -> MemorizeRequest: """ Convert POST /api/v1/memories (personal add) request to MemorizeRequest. @@ -344,7 +344,7 @@ def convert_personal_add_to_memorize_request( def convert_group_add_to_memorize_request( - request_data: Dict[str, Any] + request_data: Dict[str, Any], ) -> MemorizeRequest: """ Convert POST /api/v1/memories/group (group add) request to MemorizeRequest. @@ -436,7 +436,7 @@ def convert_group_add_to_memorize_request( def convert_personal_flush_to_memorize_request( - request_data: Dict[str, Any] + request_data: Dict[str, Any], ) -> MemorizeRequest: """ Convert POST /api/v1/memories/flush (personal flush) to MemorizeRequest. @@ -476,7 +476,7 @@ def convert_personal_flush_to_memorize_request( def convert_agent_flush_to_memorize_request( - request_data: Dict[str, Any] + request_data: Dict[str, Any], ) -> MemorizeRequest: """ Convert POST /api/v1/memories/agent/flush (agent flush) to MemorizeRequest. @@ -517,7 +517,7 @@ def convert_agent_flush_to_memorize_request( def convert_group_flush_to_memorize_request( - request_data: Dict[str, Any] + request_data: Dict[str, Any], ) -> MemorizeRequest: """ Convert POST /api/v1/memories/group/flush (group flush) to MemorizeRequest. @@ -552,7 +552,7 @@ def convert_group_flush_to_memorize_request( def convert_agent_add_to_memorize_request( - request_data: Dict[str, Any] + request_data: Dict[str, Any], ) -> MemorizeRequest: """ Convert POST /api/v1/memories/agent (agent add) request to MemorizeRequest. diff --git a/methods/EverCore/src/biz_layer/mem_memorize.py b/methods/EverCore/src/biz_layer/mem_memorize.py index 34c6d22e..de401048 100644 --- a/methods/EverCore/src/biz_layer/mem_memorize.py +++ b/methods/EverCore/src/biz_layer/mem_memorize.py @@ -92,7 +92,6 @@ def _is_agent_case_quality_sufficient( return True - async def _trigger_clustering( group_id: str, memcell: MemCell, @@ -162,12 +161,11 @@ async def _trigger_clustering( # Clustering text: task_intent for agent case, episode for normal clustering_text = ( - agent_case.task_intent if has_case and agent_case.task_intent + agent_case.task_intent + if has_case and agent_case.task_intent else episode_text ) - logger.info( - f"[Clustering] ClusterManager created (has_case={has_case})" - ) + logger.info(f"[Clustering] ClusterManager created (has_case={has_case})") # Convert MemCell to dictionary format required for clustering memcell_dict = { @@ -334,7 +332,11 @@ async def _trigger_clustering( # If you add logic that reads cluster memcells from DB here, you must # consider that new memcells may have been added between Lock 1 release # and Lock 2 acquisition. - if cluster_id and agent_case and _is_agent_case_quality_sufficient(agent_case, config): + if ( + cluster_id + and agent_case + and _is_agent_case_quality_sufficient(agent_case, config) + ): skill_lock_resource = f"trigger_agent_skill:{group_id}:{cluster_id}" async with distributed_lock( resource=skill_lock_resource, @@ -898,7 +900,11 @@ async def _clustering_with_metrics(): ) # Fire-and-forget: extract and save foresight/atomic_fact in background. # Solo scenes only; episode_saved confirms parent_doc is available for linking. - if state.is_solo_scene and state.episode_saved and not DEFAULT_MEMORIZE_CONFIG.skip_foresight_and_eventlog: + if ( + state.is_solo_scene + and state.episode_saved + and not DEFAULT_MEMORIZE_CONFIG.skip_foresight_and_eventlog + ): asyncio.create_task( _foresight_and_atomic_facts_with_metrics(state, memory_manager) ) diff --git a/methods/EverCore/src/biz_layer/memorize_config.py b/methods/EverCore/src/biz_layer/memorize_config.py index f97de895..3b7f5bfb 100644 --- a/methods/EverCore/src/biz_layer/memorize_config.py +++ b/methods/EverCore/src/biz_layer/memorize_config.py @@ -91,9 +91,12 @@ class MemorizeConfig: skip_profile_extraction=True, clustering_lock_blocking_timeout=4800, skill_extraction_lock_blocking_timeout=4800, - enable_skill_llm_verify=True + enable_skill_llm_verify=True, ) else: if _AGENT_MEMORIZE_MODE != "online": - logger.warning("Unknown AGENT_MEMORIZE_MODE=%r, falling back to 'online'", _AGENT_MEMORIZE_MODE) + logger.warning( + "Unknown AGENT_MEMORIZE_MODE=%r, falling back to 'online'", + _AGENT_MEMORIZE_MODE, + ) DEFAULT_MEMORIZE_CONFIG = MemorizeConfig() diff --git a/methods/EverCore/src/common_utils/language_utils.py b/methods/EverCore/src/common_utils/language_utils.py index 3e0ceec4..af4c7bfb 100644 --- a/methods/EverCore/src/common_utils/language_utils.py +++ b/methods/EverCore/src/common_utils/language_utils.py @@ -37,4 +37,3 @@ def is_supported_language(language: str) -> bool: 是否支持该语言 """ return language.lower() in SUPPORTED_LANGUAGES - diff --git a/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py b/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py index 9e1ff97c..85a1b47c 100644 --- a/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py +++ b/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py @@ -97,10 +97,12 @@ def deserialize_data(data: Union[str, bytes]) -> Any: # Try decoding as string for JSON deserialization try: data_str = data.decode('utf-8') - return json.loads(data_str) except UnicodeDecodeError: logger.warning("Binary data cannot be decoded as UTF-8") return data + + try: + return json.loads(data_str) except json.JSONDecodeError: # JSON parsing failed, but UTF-8 decoding succeeded, return decoded string logger.debug( @@ -181,7 +183,7 @@ def parse_member_data(member: Union[str, bytes]) -> Tuple[str, Union[str, bytes] @staticmethod def process_data_for_storage( - data: Union[str, Dict, List, Any] + data: Union[str, Dict, List, Any], ) -> Union[str, bytes]: """ Process data for storage diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py index b5ce597b..80bfc269 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py @@ -4,10 +4,7 @@ from typing import Dict, Any, List, Union, AsyncGenerator import os from google.genai.client import Client -from google.genai.types import ( - GenerateContentConfig, - ContentDict, -) +from google.genai.types import GenerateContentConfig, ContentDict from google.genai.types import ThinkingConfig from core.component.llm.llm_adapter.completion import ( ChatCompletionRequest, @@ -93,13 +90,13 @@ def _convert_messages_to_gemini_format( """Convert message list to Gemini format""" contents = [] for msg in messages: - if type(msg) == HumanMessage: + if isinstance(msg, HumanMessage): contents.append(ContentDict(role="user", parts=[{"text": msg.content}])) - elif type(msg) == AIMessage: + elif isinstance(msg, AIMessage): contents.append( ContentDict(role="model", parts=[{"text": msg.content}]) ) - elif type(msg) == SystemMessage: + elif isinstance(msg, SystemMessage): contents.append( ContentDict(role="model", parts=[{"text": msg.content}]) ) diff --git a/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py b/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py index 70aa1933..23c34fcc 100644 --- a/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py +++ b/methods/EverCore/src/core/component/llm/tokenizer/tokenizer_factory.py @@ -16,7 +16,7 @@ # Default tiktoken encodings to preload during application startup DEFAULT_TIKTOKEN_ENCODINGS = [ - "o200k_base", # GPT-4o, GPT-4o-mini + "o200k_base", # GPT-4o, GPT-4o-mini "cl100k_base", # GPT-4, GPT-3.5-turbo, text-embedding-ada-002 ] @@ -67,14 +67,20 @@ def load_default_encodings(self) -> None: The encodings loaded are defined in DEFAULT_TIKTOKEN_ENCODINGS. """ - logger.info("Preloading %d tiktoken encodings...", len(DEFAULT_TIKTOKEN_ENCODINGS)) + logger.info( + "Preloading %d tiktoken encodings...", len(DEFAULT_TIKTOKEN_ENCODINGS) + ) for encoding_name in DEFAULT_TIKTOKEN_ENCODINGS: try: self.get_tokenizer_from_tiktoken(encoding_name) - logger.info("Successfully preloaded tiktoken encoding: %s", encoding_name) + logger.info( + "Successfully preloaded tiktoken encoding: %s", encoding_name + ) except (ValueError, KeyError, RuntimeError) as e: - logger.error("Failed to preload tiktoken encoding '%s': %s", encoding_name, e) + logger.error( + "Failed to preload tiktoken encoding '%s': %s", encoding_name, e + ) logger.info("Tiktoken encodings preload completed") diff --git a/methods/EverCore/src/core/di/container.py b/methods/EverCore/src/core/di/container.py index fcdd1e3d..04d3c0a1 100644 --- a/methods/EverCore/src/core/di/container.py +++ b/methods/EverCore/src/core/di/container.py @@ -432,8 +432,8 @@ def _build_inheritance_cache(self): for base in registered_type.__mro__[1:]: # Skip self # Exclude abc.ABC base class and object base class, they are too generic if ( - base != abc.ABC - and base != object + base is not abc.ABC + and base is not object and hasattr(base, '__abstractmethods__') ): # ABC type all_parent_types.add(base) diff --git a/methods/EverCore/src/core/di/tests/test_di_scanner.py b/methods/EverCore/src/core/di/tests/test_di_scanner.py index 90a4ae45..0d920380 100644 --- a/methods/EverCore/src/core/di/tests/test_di_scanner.py +++ b/methods/EverCore/src/core/di/tests/test_di_scanner.py @@ -13,10 +13,7 @@ from core.di.scanner import ComponentScanner from core.di.decorators import component, service, repository, mock_impl, factory from core.di.bean_definition import BeanScope -from core.di.tests.test_fixtures import ( - UserRepository, - MySQLUserRepository, -) +from core.di.tests.test_fixtures import UserRepository, MySQLUserRepository class TestScannerBasicFunctionality: diff --git a/methods/EverCore/src/core/interface/controller/base_controller.py b/methods/EverCore/src/core/interface/controller/base_controller.py index fd9ffcb2..ba55500d 100644 --- a/methods/EverCore/src/core/interface/controller/base_controller.py +++ b/methods/EverCore/src/core/interface/controller/base_controller.py @@ -112,9 +112,9 @@ def __init__( self.router = APIRouter(prefix=prefix, tags=tags, **kwargs) self._app: Optional[FastAPI] = None self._extra_models: List[Any] = [] - self._auth_routes: List[str] = ( - [] - ) # Store paths of routes requiring authentication + self._auth_routes: List[ + str + ] = [] # Store paths of routes requiring authentication self._default_auth = default_auth # Store default authorization strategy self._collect_routes() @@ -539,9 +539,9 @@ def _process_controller_extra_models(self, controller, openapi_schema): ) del arg_schema['$defs'] # Add main model schema - openapi_schema["components"]["schemas"][ - arg_name - ] = arg_schema + openapi_schema["components"]["schemas"][arg_name] = ( + arg_schema + ) else: # Process regular model model_name = self._get_model_name(model) @@ -557,9 +557,9 @@ def _process_controller_extra_models(self, controller, openapi_schema): ) del model_schema['$defs'] # Add main model schema - openapi_schema["components"]["schemas"][ - model_name - ] = model_schema + openapi_schema["components"]["schemas"][model_name] = ( + model_schema + ) def register_to_app(self, app: FastAPI): """ diff --git a/methods/EverCore/src/core/lifespan/database_lifespan.py b/methods/EverCore/src/core/lifespan/database_lifespan.py index 31ed0781..9d40ed3e 100644 --- a/methods/EverCore/src/core/lifespan/database_lifespan.py +++ b/methods/EverCore/src/core/lifespan/database_lifespan.py @@ -45,9 +45,10 @@ async def startup(self, app: FastAPI) -> Tuple[Any, Any, Any]: self._db_provider = get_bean_by_type(DatabaseConnectionProvider) # Get connection pool and checkpointer - pool, checkpointer = ( - await self._db_provider.get_connection_and_checkpointer() - ) + ( + pool, + checkpointer, + ) = await self._db_provider.get_connection_and_checkpointer() # Store connection pool and checkpointer in app.state for business logic usage app.state.connection_pool = pool diff --git a/methods/EverCore/src/core/lifespan/metrics_lifespan.py b/methods/EverCore/src/core/lifespan/metrics_lifespan.py index 85a74c36..25d20d44 100644 --- a/methods/EverCore/src/core/lifespan/metrics_lifespan.py +++ b/methods/EverCore/src/core/lifespan/metrics_lifespan.py @@ -3,6 +3,7 @@ Starts standalone Prometheus metrics server on a separate port (default: 9090). """ + import os from fastapi import FastAPI from typing import Any, Tuple @@ -72,4 +73,3 @@ async def shutdown(self, app: FastAPI) -> None: # Clean up app.state if hasattr(app.state, 'metrics_port'): delattr(app.state, 'metrics_port') - diff --git a/methods/EverCore/src/core/lifespan/mongodb_lifespan.py b/methods/EverCore/src/core/lifespan/mongodb_lifespan.py index 0b0f4015..d131b66a 100644 --- a/methods/EverCore/src/core/lifespan/mongodb_lifespan.py +++ b/methods/EverCore/src/core/lifespan/mongodb_lifespan.py @@ -12,7 +12,10 @@ from core.lifespan.lifespan_interface import LifespanProvider from core.oxm.mongo.document_base import DocumentBase from core.di.utils import get_all_subclasses -from core.component.mongodb_client_factory import MongoDBClientFactory, MongoDBClientWrapper +from core.component.mongodb_client_factory import ( + MongoDBClientFactory, + MongoDBClientWrapper, +) logger = get_logger(__name__) @@ -47,7 +50,6 @@ async def startup(self, app: FastAPI) -> Any: logger.info("Initializing MongoDB connection...") try: - # Get MongoDB client factory self._mongodb_factory = get_bean_by_type(MongoDBClientFactory) diff --git a/methods/EverCore/src/core/lock/redis_distributed_lock.py b/methods/EverCore/src/core/lock/redis_distributed_lock.py index 051d1d60..bf0c8552 100644 --- a/methods/EverCore/src/core/lock/redis_distributed_lock.py +++ b/methods/EverCore/src/core/lock/redis_distributed_lock.py @@ -78,9 +78,7 @@ async def acquire( if acquired: try: # Call the lock manager's internal method to release the lock - await self.lock_manager._release_lock( - self.resource - ) # pylint: disable=protected-access + await self.lock_manager._release_lock(self.resource) # pylint: disable=protected-access self._acquired = False except (ConnectionError, TimeoutError, OSError) as e: logger.error( diff --git a/methods/EverCore/src/core/middleware/database_session_middleware.py b/methods/EverCore/src/core/middleware/database_session_middleware.py index e3f7b69a..75f8c932 100644 --- a/methods/EverCore/src/core/middleware/database_session_middleware.py +++ b/methods/EverCore/src/core/middleware/database_session_middleware.py @@ -6,10 +6,7 @@ from typing import Callable, AsyncGenerator from sqlmodel.ext.asyncio.session import AsyncSession -from core.context.context import ( - set_current_session, - clear_current_session, -) +from core.context.context import set_current_session, clear_current_session from core.component.database_session_provider import DatabaseSessionProvider from core.di.utils import get_bean_by_type from core.observation.logger import get_logger diff --git a/methods/EverCore/src/core/middleware/prometheus_middleware.py b/methods/EverCore/src/core/middleware/prometheus_middleware.py index 1f2f2f88..ca9845d6 100644 --- a/methods/EverCore/src/core/middleware/prometheus_middleware.py +++ b/methods/EverCore/src/core/middleware/prometheus_middleware.py @@ -11,6 +11,7 @@ app = FastAPI() app.add_middleware(PrometheusMiddleware) """ + from core.observation.logger import get_logger import time from typing import Callable @@ -116,7 +117,6 @@ def _normalize_path(request: Request) -> str: return '{unmatched}' - class PrometheusMiddleware(BaseHTTPMiddleware): """ Prometheus HTTP Metrics Middleware @@ -169,25 +169,27 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: # Record metrics duration = time.perf_counter() - start_time - _http_requests_total.labels( - method=method, - path=path, - status=status, - ).inc() + _http_requests_total.labels(method=method, path=path, status=status).inc() - _http_request_duration_seconds.labels( - method=method, - path=path, - ).observe(duration) + _http_request_duration_seconds.labels(method=method, path=path).observe( + duration + ) # Record request size if request_size > 0: - _http_request_size_bytes.labels(method=method, path=path).observe(request_size) + _http_request_size_bytes.labels(method=method, path=path).observe( + request_size + ) # Record response size - if response and hasattr(response, 'headers') and response.headers.get('content-length'): + if ( + response + and hasattr(response, 'headers') + and response.headers.get('content-length') + ): response_size = int(response.headers.get('content-length', 0)) - _http_response_size_bytes.labels(method=method, path=path).observe(response_size) + _http_response_size_bytes.labels(method=method, path=path).observe( + response_size + ) return response - diff --git a/methods/EverCore/src/core/middleware/sse_exception_middleware.py b/methods/EverCore/src/core/middleware/sse_exception_middleware.py index 0563a5a3..604c221d 100644 --- a/methods/EverCore/src/core/middleware/sse_exception_middleware.py +++ b/methods/EverCore/src/core/middleware/sse_exception_middleware.py @@ -31,7 +31,7 @@ def yield_sse_data(data: Any) -> str: def sse_exception_handler( - func: Callable[..., AsyncGenerator[str, None]] + func: Callable[..., AsyncGenerator[str, None]], ) -> Callable[..., AsyncGenerator[str, None]]: """ SSE stream exception handling decorator diff --git a/methods/EverCore/src/core/observation/logger.py b/methods/EverCore/src/core/observation/logger.py index d5b5c1aa..f86a0865 100644 --- a/methods/EverCore/src/core/observation/logger.py +++ b/methods/EverCore/src/core/observation/logger.py @@ -63,7 +63,7 @@ def _setup_root_logging(self): level=getattr(logging, log_level), format='%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - [%(request_id)s] - %(message)s', handlers=[ - logging.StreamHandler(sys.stdout), + logging.StreamHandler(sys.stdout) # Can add file handler # logging.FileHandler('app.log', encoding='utf-8') ], diff --git a/methods/EverCore/src/core/observation/metrics/__init__.py b/methods/EverCore/src/core/observation/metrics/__init__.py index e571313a..9b7474da 100644 --- a/methods/EverCore/src/core/observation/metrics/__init__.py +++ b/methods/EverCore/src/core/observation/metrics/__init__.py @@ -42,35 +42,26 @@ def refresh(self, labels: dict) -> float: generate_metrics_response, reset_metrics_registry, ) -from .server import ( - start_metrics_server, - is_metrics_server_running, - get_metrics_url, -) +from .server import start_metrics_server, is_metrics_server_running, get_metrics_url __all__ = [ # Counter 'Counter', 'LabeledCounter', - # Histogram 'Histogram', 'LabeledHistogram', 'HistogramBuckets', - # Gauge 'BaseGauge', 'LabeledGauge', - # Registry 'get_metrics_registry', 'set_metrics_registry', 'generate_metrics_response', 'reset_metrics_registry', - # Server 'start_metrics_server', 'is_metrics_server_running', 'get_metrics_url', ] - diff --git a/methods/EverCore/src/core/observation/metrics/counter.py b/methods/EverCore/src/core/observation/metrics/counter.py index 13e328b7..50c36797 100644 --- a/methods/EverCore/src/core/observation/metrics/counter.py +++ b/methods/EverCore/src/core/observation/metrics/counter.py @@ -3,6 +3,7 @@ Provides a unified Counter interface, isolating prometheus_client from business code. """ + from prometheus_client import Counter as PrometheusCounter from typing import Sequence from .registry import get_metrics_registry @@ -96,4 +97,3 @@ def inc(self, amount: float = 1) -> None: amount: Increment amount, defaults to 1 """ self._counter.inc(amount) - diff --git a/methods/EverCore/src/core/observation/metrics/gauge.py b/methods/EverCore/src/core/observation/metrics/gauge.py index d8951f18..eca5809d 100644 --- a/methods/EverCore/src/core/observation/metrics/gauge.py +++ b/methods/EverCore/src/core/observation/metrics/gauge.py @@ -3,6 +3,7 @@ Provides a unified Gauge interface with built-in auto-refresh capability. """ + from prometheus_client import Gauge as PrometheusGauge from typing import Sequence, Optional, Callable, Any, Dict, Tuple import asyncio @@ -68,6 +69,7 @@ def __init__( unit: Unit (optional) """ from .registry import get_metrics_registry + registry = get_metrics_registry() self._gauge = PrometheusGauge( @@ -192,9 +194,7 @@ def set_to_current_time(self) -> None: self._labeled_gauge.set_to_current_time() def start_refresh( - self, - interval_seconds: int = 5, - enable_async: bool = True, + self, interval_seconds: int = 5, enable_async: bool = True ) -> 'LabeledGauge': """ Start auto-refresh @@ -223,9 +223,7 @@ async def refresh(self, labels: dict) -> float: # Stop existing task if any (prevent task leak) existing_task = self._base_gauge._refresh_tasks.get(self._label_key) if existing_task and existing_task._running: - logger.warning( - f"Replacing existing refresh task for {self._label_key}" - ) + logger.warning(f"Replacing existing refresh task for {self._label_key}") # Schedule stop in background to avoid blocking asyncio.create_task(existing_task.stop()) @@ -319,8 +317,8 @@ async def _refresh_loop(self) -> None: try: # Check if it's an async function if self.enable_async and ( - asyncio.iscoroutinefunction(self.refresh_func) or - inspect.iscoroutinefunction(self.refresh_func) + asyncio.iscoroutinefunction(self.refresh_func) + or inspect.iscoroutinefunction(self.refresh_func) ): value = await self.refresh_func() else: @@ -339,7 +337,7 @@ async def _refresh_loop(self) -> None: logger.error( f"Refresh failed for {self.label_key}: {e} " f"(error_count={self._error_count})", - exc_info=True + exc_info=True, ) # Wait for next refresh diff --git a/methods/EverCore/src/core/observation/metrics/histogram.py b/methods/EverCore/src/core/observation/metrics/histogram.py index 880b0317..578fe59a 100644 --- a/methods/EverCore/src/core/observation/metrics/histogram.py +++ b/methods/EverCore/src/core/observation/metrics/histogram.py @@ -3,6 +3,7 @@ Provides a unified Histogram interface, isolating prometheus_client from business code. """ + from prometheus_client import Histogram as PrometheusHistogram from typing import Sequence from .registry import get_metrics_registry @@ -14,8 +15,20 @@ class HistogramBuckets: # Default buckets (covering 5ms - 10s) DEFAULT = ( - 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, - 0.75, 1.0, 2.5, 5.0, 7.5, 10.0 + 0.005, + 0.01, + 0.025, + 0.05, + 0.075, + 0.1, + 0.25, + 0.5, + 0.75, + 1.0, + 2.5, + 5.0, + 7.5, + 10.0, ) # Fast operations (5ms - 500ms, for cache queries, simple calculations, etc.) @@ -23,14 +36,44 @@ class HistogramBuckets: # API calls (10ms - 30s, for external API calls) # Denser buckets in 0.1-5s range for better P95/P99 accuracy - API_CALL = (0.01, 0.05, 0.1, 0.2, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0, 30.0) + API_CALL = ( + 0.01, + 0.05, + 0.1, + 0.2, + 0.3, + 0.5, + 0.75, + 1.0, + 1.5, + 2.0, + 3.0, + 5.0, + 10.0, + 30.0, + ) # Batch operations (100ms - 60s, for batch processing) BATCH = (0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0) # Embedding/Rerank (10ms - 10s, for ML inference) # Denser buckets in 0.1-3s range where most requests fall - ML_INFERENCE = (0.01, 0.025, 0.05, 0.1, 0.2, 0.3, 0.5, 0.75, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0) + ML_INFERENCE = ( + 0.01, + 0.025, + 0.05, + 0.1, + 0.2, + 0.3, + 0.5, + 0.75, + 1.0, + 1.5, + 2.0, + 3.0, + 5.0, + 10.0, + ) # Database queries (1ms - 5s) DATABASE = (0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0) @@ -152,4 +195,3 @@ def time(self): do_something() """ return self._histogram.time() - diff --git a/methods/EverCore/src/core/observation/metrics/registry.py b/methods/EverCore/src/core/observation/metrics/registry.py index 38f75de7..0a8d9958 100644 --- a/methods/EverCore/src/core/observation/metrics/registry.py +++ b/methods/EverCore/src/core/observation/metrics/registry.py @@ -3,6 +3,7 @@ Centralized management of Prometheus metrics registry with singleton access. """ + from prometheus_client import CollectorRegistry, REGISTRY, generate_latest from typing import Optional import logging @@ -61,4 +62,3 @@ def reset_metrics_registry() -> None: global _metrics_registry _metrics_registry = None logger.warning("Metrics registry has been reset") - diff --git a/methods/EverCore/src/core/observation/metrics/server.py b/methods/EverCore/src/core/observation/metrics/server.py index 588e2779..a8516bd3 100644 --- a/methods/EverCore/src/core/observation/metrics/server.py +++ b/methods/EverCore/src/core/observation/metrics/server.py @@ -18,6 +18,7 @@ - Availability: Metrics available even if main app is overloaded - Operations: Can expose to internal network only """ + import os import logging from typing import Optional @@ -30,10 +31,7 @@ _metrics_server_started: bool = False -def start_metrics_server( - port: Optional[int] = None, - addr: str = "0.0.0.0", -) -> bool: +def start_metrics_server(port: Optional[int] = None, addr: str = "0.0.0.0") -> bool: """ Start standalone Prometheus metrics HTTP server @@ -66,11 +64,7 @@ def start_metrics_server( try: # Start HTTP server using prometheus_client's built-in server # This creates a daemon thread that serves /metrics endpoint - start_http_server( - port=port, - addr=addr, - registry=get_metrics_registry(), - ) + start_http_server(port=port, addr=addr, registry=get_metrics_registry()) _metrics_server_started = True logger.info(f"✅ Metrics server started on {addr}:{port}/metrics") @@ -100,4 +94,3 @@ def get_metrics_url(host: str = "localhost", port: Optional[int] = None) -> str: if port is None: port = int(os.getenv("METRICS_PORT", "9090")) return f"http://{host}:{port}/metrics" - diff --git a/methods/EverCore/src/core/oxm/constants.py b/methods/EverCore/src/core/oxm/constants.py index 72b0aebc..a2c04fe8 100644 --- a/methods/EverCore/src/core/oxm/constants.py +++ b/methods/EverCore/src/core/oxm/constants.py @@ -17,8 +17,4 @@ MAX_RETRIEVE_LIMIT = 500 # Export all constants -__all__ = [ - "MAGIC_ALL", - "MAX_FETCH_LIMIT", - "MAX_RETRIEVE_LIMIT", -] +__all__ = ["MAGIC_ALL", "MAX_FETCH_LIMIT", "MAX_RETRIEVE_LIMIT"] diff --git a/methods/EverCore/src/core/oxm/es/analyzer.py b/methods/EverCore/src/core/oxm/es/analyzer.py index e0396d27..7e2639d5 100644 --- a/methods/EverCore/src/core/oxm/es/analyzer.py +++ b/methods/EverCore/src/core/oxm/es/analyzer.py @@ -99,7 +99,9 @@ # For example: "running", "runs", "ran" -> "run" # "better", "good" -> "good", "better" (irregular forms require special handling) snow_en_filter = token_filter( - "snow_filter", type="snowball", language="English" # English stemming + "snow_filter", + type="snowball", + language="English", # English stemming ) # English stemming analyzer - used for semantic search on English text diff --git a/methods/EverCore/src/core/oxm/milvus/migration/utils.py b/methods/EverCore/src/core/oxm/milvus/migration/utils.py index b2ad1528..6d4a4da0 100644 --- a/methods/EverCore/src/core/oxm/milvus/migration/utils.py +++ b/methods/EverCore/src/core/oxm/milvus/migration/utils.py @@ -67,14 +67,10 @@ def find_collection_manager_by_alias(alias: str) -> Type[MilvusCollectionBase]: # Temporarily instantiate to get alias (requires parsing suffix) try: # Try to parse suffix from alias - base_name = ( - doc_class._COLLECTION_NAME - ) # pylint: disable=protected-access + base_name = doc_class._COLLECTION_NAME # pylint: disable=protected-access if alias.startswith(base_name): return doc_class - except ( - Exception - ): # pylint: disable=broad-except # Ignore instantiation failure, continue to next class + except Exception: # pylint: disable=broad-except # Ignore instantiation failure, continue to next class continue else: # For MilvusCollectionBase, directly compare _COLLECTION_NAME @@ -120,9 +116,7 @@ def rebuild_collection( # 2. Instantiate manager (parse suffix from alias) if issubclass(collection_class, MilvusCollectionWithSuffix): # Parse suffix from alias - base_name = ( - collection_class._COLLECTION_NAME - ) # pylint: disable=protected-access + base_name = collection_class._COLLECTION_NAME # pylint: disable=protected-access suffix = None if alias != base_name and alias.startswith(base_name + "_"): suffix = alias[len(base_name) + 1 :] diff --git a/methods/EverCore/src/core/oxm/mongo/__init__.py b/methods/EverCore/src/core/oxm/mongo/__init__.py index 5cfe2ee0..ad302732 100755 --- a/methods/EverCore/src/core/oxm/mongo/__init__.py +++ b/methods/EverCore/src/core/oxm/mongo/__init__.py @@ -8,8 +8,4 @@ from core.oxm.mongo.document_base_with_soft_delete import DocumentBaseWithSoftDelete from core.oxm.mongo.audit_base import AuditBase -__all__ = [ - "DocumentBase", - "DocumentBaseWithSoftDelete", - "AuditBase", -] +__all__ = ["DocumentBase", "DocumentBaseWithSoftDelete", "AuditBase"] diff --git a/methods/EverCore/src/core/oxm/mongo/document_base.py b/methods/EverCore/src/core/oxm/mongo/document_base.py index 9a6dbb3c..27ab0376 100644 --- a/methods/EverCore/src/core/oxm/mongo/document_base.py +++ b/methods/EverCore/src/core/oxm/mongo/document_base.py @@ -153,7 +153,6 @@ def check_datetimes_are_aware(self) -> Self: for field_name, value in self: new_value = self._recursive_datetime_check(value, field_name, depth=0) if new_value is not value: # Only update if value has changed - # Directly update value using __dict__ to avoid triggering validators self.__dict__[field_name] = new_value return self diff --git a/methods/EverCore/src/core/oxm/pg/audit_base.py b/methods/EverCore/src/core/oxm/pg/audit_base.py index 117331fd..159bcc69 100644 --- a/methods/EverCore/src/core/oxm/pg/audit_base.py +++ b/methods/EverCore/src/core/oxm/pg/audit_base.py @@ -77,9 +77,7 @@ def is_deleted(self) -> bool: # Register event listeners @event.listens_for(AuditableModel, 'before_insert', propagate=True) - def before_insert_listener( - mapper, connection, target - ): # pylint: disable=unused-argument + def before_insert_listener(mapper, connection, target): # pylint: disable=unused-argument """Event listener before INSERT operation""" # Ignore unused parameters (required signature for SQLAlchemy event listeners) _ = mapper, connection @@ -102,9 +100,7 @@ def before_insert_listener( target.updated_by = current_user_id or "system" @event.listens_for(AuditableModel, 'before_update', propagate=True) - def before_update_listener( - mapper, connection, target - ): # pylint: disable=unused-argument + def before_update_listener(mapper, connection, target): # pylint: disable=unused-argument """Event listener before UPDATE operation""" # Ignore unused parameters (required signature for SQLAlchemy event listeners) _ = mapper, connection diff --git a/methods/EverCore/src/core/queue/msg_group_queue/msg_group_queue_manager.py b/methods/EverCore/src/core/queue/msg_group_queue/msg_group_queue_manager.py index 20cb45e0..c05ce229 100644 --- a/methods/EverCore/src/core/queue/msg_group_queue/msg_group_queue_manager.py +++ b/methods/EverCore/src/core/queue/msg_group_queue/msg_group_queue_manager.py @@ -297,7 +297,7 @@ async def get_by_queue( """ if queue_id < 0 or queue_id >= self.num_queues: raise ValueError( - f"Queue ID out of range: {queue_id}, valid range: 0-{self.num_queues-1}" + f"Queue ID out of range: {queue_id}, valid range: 0-{self.num_queues - 1}" ) target_queue = self._queues[queue_id] @@ -387,7 +387,7 @@ async def get_queue_info( if queue_id is not None: if queue_id < 0 or queue_id >= self.num_queues: raise ValueError( - f"Queue ID out of range: {queue_id}, valid range: 0-{self.num_queues-1}" + f"Queue ID out of range: {queue_id}, valid range: 0-{self.num_queues - 1}" ) return self._queue_stats[queue_id].to_dict() else: @@ -467,17 +467,25 @@ def _update_time_window_stats(self): # Update time window stats for each queue for i in range(self.num_queues): # Clean old events and count - self._queue_stats[i].time_window_stats.delivered_1min = ( - self._count_events_in_window(self._delivery_events[i], 60.0) + self._queue_stats[ + i + ].time_window_stats.delivered_1min = self._count_events_in_window( + self._delivery_events[i], 60.0 ) - self._queue_stats[i].time_window_stats.consumed_1min = ( - self._count_events_in_window(self._consume_events[i], 60.0) + self._queue_stats[ + i + ].time_window_stats.consumed_1min = self._count_events_in_window( + self._consume_events[i], 60.0 ) - self._queue_stats[i].time_window_stats.delivered_1hour = ( - self._count_events_in_window(self._delivery_events[i], 3600.0) + self._queue_stats[ + i + ].time_window_stats.delivered_1hour = self._count_events_in_window( + self._delivery_events[i], 3600.0 ) - self._queue_stats[i].time_window_stats.consumed_1hour = ( - self._count_events_in_window(self._consume_events[i], 3600.0) + self._queue_stats[ + i + ].time_window_stats.consumed_1hour = self._count_events_in_window( + self._consume_events[i], 3600.0 ) # Update manager time window statistics diff --git a/methods/EverCore/src/core/request/__init__.py b/methods/EverCore/src/core/request/__init__.py index b2ebe49f..eec0c535 100644 --- a/methods/EverCore/src/core/request/__init__.py +++ b/methods/EverCore/src/core/request/__init__.py @@ -36,4 +36,3 @@ 'log_request_default', 'RequestHistoryEvent', ] - diff --git a/methods/EverCore/src/core/request/timeout_background.py b/methods/EverCore/src/core/request/timeout_background.py index 2c729e5d..80e6c4b0 100644 --- a/methods/EverCore/src/core/request/timeout_background.py +++ b/methods/EverCore/src/core/request/timeout_background.py @@ -114,7 +114,7 @@ async def memorize(request: MemorizeRequest): """ def decorator( - func: Callable[P, Coroutine[Any, Any, T]] + func: Callable[P, Coroutine[Any, Any, T]], ) -> Callable[P, Coroutine[Any, Any, Union[T, JSONResponse]]]: @wraps(func) diff --git a/methods/EverCore/src/core/tenants/tenant_config.py b/methods/EverCore/src/core/tenants/tenant_config.py index 8ed3228c..e0ad654b 100644 --- a/methods/EverCore/src/core/tenants/tenant_config.py +++ b/methods/EverCore/src/core/tenants/tenant_config.py @@ -29,9 +29,7 @@ class TenantConfig: def __init__(self): """Initialize tenant configuration""" self._single_tenant_id: Optional[str] = None - self._app_ready: bool = ( - False # Application startup completion status, used for strict tenant checks - ) + self._app_ready: bool = False # Application startup completion status, used for strict tenant checks @property def single_tenant_id(self) -> Optional[str]: diff --git a/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py b/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py index cbedb3b4..e83c1a62 100644 --- a/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py +++ b/methods/EverCore/src/core/tenants/tenantize/kv/redis/tenant_key_utils.py @@ -5,8 +5,6 @@ """ - - def build_tenant_redis_key(prefix: str, tenant_id: str, key: str) -> str: """ Build a tenant-scoped Redis key with an explicit tenant_id. diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py b/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py index c05e49e0..67d34ddb 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_field_es_interceptor.py @@ -27,16 +27,7 @@ → raise TenantIsolationViolation """ -from typing import ( - Any, - Collection, - Dict, - FrozenSet, - Mapping, - Optional, - Tuple, - Union, -) +from typing import Any, Collection, Dict, FrozenSet, Mapping, Optional, Tuple, Union from elasticsearch import AsyncElasticsearch from elastic_transport import ( diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py index d8efe390..e06df552 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py @@ -477,7 +477,9 @@ def compute_database_name() -> str: return get_or_compute_tenant_cache( patch_key=TenantPatchKey.ACTUAL_DATABASE_NAME, compute_func=compute_database_name, - fallback=lambda: generate_tenant_database_name(), # Lazy: returns b0001_memsys when no tenant context + fallback=lambda: ( + generate_tenant_database_name() + ), # Lazy: returns b0001_memsys when no tenant context cache_description="database name", ) diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_field_command_interceptor.py b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_field_command_interceptor.py index e92fde60..c1244f8e 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_field_command_interceptor.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_field_command_interceptor.py @@ -555,8 +555,7 @@ def install_tenant_interceptor( ) logger.info( - "Tenant command interceptor installed on client %s " - "(excluded_collections=%s)", + "Tenant command interceptor installed on client %s (excluded_collections=%s)", type(client).__name__, excluded_collections or "none", ) diff --git a/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py b/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py index be95979a..816d8b82 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py +++ b/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py @@ -238,6 +238,7 @@ def run( try: # Determine whether to pass the callback function based on whether data migration is needed if migrate_data: + def populate_fn(old_col, new_col): return migrate_data_callback( old_col, new_col, batch_size, progress=progress, alias=alias diff --git a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py index 6ba3a8ac..ba3eddd8 100644 --- a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py +++ b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py @@ -654,7 +654,7 @@ async def translate_file( file_size = file_path.stat().st_size if file_size > MAX_FILE_SIZE: print( - f"{progress_prefix} [SKIP-LARGE] {file_path} - File too large ({file_size/1024:.1f}KB)" + f"{progress_prefix} [SKIP-LARGE] {file_path} - File too large ({file_size / 1024:.1f}KB)" ) async with progress_lock: progress["processed"].append(file_str) @@ -672,7 +672,7 @@ async def translate_file( return (file_path, True, None) print( - f"{progress_prefix} [TRANSLATING] {file_path} ({file_size/1024:.1f}KB)" + f"{progress_prefix} [TRANSLATING] {file_path} ({file_size / 1024:.1f}KB)" ) prompt = TRANSLATION_PROMPT.format(code=original_content) @@ -748,7 +748,7 @@ async def review_file_diff( result = FileReviewResult( file_path=file_path, result=ReviewResult.NEEDS_REVIEW, - reason=f"Diff too large for automated analysis ({len(diff)/1024:.1f}KB > {MAX_DIFF_SIZE/1024:.0f}KB limit)", + reason=f"Diff too large for automated analysis ({len(diff) / 1024:.1f}KB > {MAX_DIFF_SIZE / 1024:.0f}KB limit)", diff_summary=f"Diff size: {len(diff) / 1024:.1f}KB", ) async with progress_lock: @@ -757,7 +757,7 @@ async def review_file_diff( ) save_review_progress(progress) print( - f"{progress_prefix} [NEEDS-REVIEW] {file_path} - Diff too large ({len(diff)/1024:.1f}KB)" + f"{progress_prefix} [NEEDS-REVIEW] {file_path} - Diff too large ({len(diff) / 1024:.1f}KB)" ) return result @@ -876,7 +876,7 @@ async def cmd_translate( total_files = len(python_files) print(f"Found {total_files} Python files in total") print(f"Max concurrency: {max_concurrency}") - print(f"Max file size: {MAX_FILE_SIZE/1024:.1f}KB") + print(f"Max file size: {MAX_FILE_SIZE / 1024:.1f}KB") print(f"Dry run: {dry_run}") print() @@ -1137,11 +1137,11 @@ async def cmd_review( # If diff is too large, try with minimal context (0 lines) if success and len(diff) > MAX_DIFF_SIZE: print( - f" [INFO] {file_path}: diff too large ({len(diff)/1024:.1f}KB), retrying with minimal context..." + f" [INFO] {file_path}: diff too large ({len(diff) / 1024:.1f}KB), retrying with minimal context..." ) success, diff = get_file_diff(commit_ref, file_path, context_lines=0) if success: - print(f" [INFO] {file_path}: reduced to {len(diff)/1024:.1f}KB") + print(f" [INFO] {file_path}: reduced to {len(diff) / 1024:.1f}KB") if not success: diff --git a/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py b/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py index a6b2fd0e..1233add3 100644 --- a/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py +++ b/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py @@ -76,9 +76,9 @@ def show_collection_details(client: MilvusClient, names: List[str]) -> None: print("\nNo collections found.") return - print(f"\n{'='*80}") + print(f"\n{'=' * 80}") print(f" Found {len(names)} collection(s)") - print(f"{'='*80}\n") + print(f"{'=' * 80}\n") header = f"{'#':<4} {'Collection Name':<50} {'Rows':>10} {'Aliases'}" print(header) diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/dto/memory_dto.py b/methods/EverCore/src/infra_layer/adapters/input/api/dto/memory_dto.py index 53120160..64bfd5b4 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/dto/memory_dto.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/dto/memory_dto.py @@ -32,6 +32,7 @@ # Backward compatibility aliases SearchMemoriesRequest = RetrieveMemRequest +DeleteMemoriesRequest = DeleteMemoriesRequestDTO __all__ = [ # Base Response diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py index 760d050f..02166256 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py @@ -90,10 +90,7 @@ async def get_cluster_assignments(self, group_id: str) -> Dict[str, str]: return {} # Derive eventid_to_cluster from memcell_info memcell_info = mem_scene.memcell_info or {} - return { - eid: info.get("memscene", "") - for eid, info in memcell_info.items() - } + return {eid: info.get("memscene", "") for eid, info in memcell_info.items()} except Exception as e: logger.error( f"Failed to retrieve cluster assignments: group_id={group_id}, error={e}" diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py index d9ad9c8d..e2954bd5 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py @@ -16,9 +16,7 @@ from core.oxm.mongo.base_repository import BaseRepository from core.oxm.constants import MAGIC_ALL -from infra_layer.adapters.out.persistence.document.memory.memcell import ( - MemCell, -) +from infra_layer.adapters.out.persistence.document.memory.memcell import MemCell logger = get_logger(__name__) diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py index 65dd41e7..58718a53 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py @@ -326,7 +326,9 @@ async def upsert( # Trigger Milvus indexing (runs in clustering background task, not on hot path) if trigger_index: - await self._trigger_milvus_indexing(user_id, group_id, profile_data, doc_id=str(saved_profile.id)) + await self._trigger_milvus_indexing( + user_id, group_id, profile_data, doc_id=str(saved_profile.id) + ) return saved_profile @@ -337,7 +339,11 @@ async def upsert( return None async def _trigger_milvus_indexing( - self, user_id: str, group_id: str, profile_data: Dict[str, Any], doc_id: str = "" + self, + user_id: str, + group_id: str, + profile_data: Dict[str, Any], + doc_id: str = "", ) -> None: """ Trigger Milvus indexing for profile (delete-then-insert strategy) diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py index ea5299b2..c3c6018e 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py @@ -91,7 +91,9 @@ def _make_entity(embed_text: str, item_type: str) -> Dict[str, Any]: for field_name, label in _EXPLICIT_FIELDS: for item in profile_data.get(field_name, []) or []: - value = item.get("value", "") if isinstance(item, dict) else str(item) + value = ( + item.get("value", "") if isinstance(item, dict) else str(item) + ) if not value: continue level = item.get("level", "") if isinstance(item, dict) else "" @@ -100,7 +102,9 @@ def _make_entity(embed_text: str, item_type: str) -> Dict[str, Any]: for field_name, label in _IMPLICIT_FIELDS: for item in profile_data.get(field_name, []) or []: - value = item.get("value", "") if isinstance(item, dict) else str(item) + value = ( + item.get("value", "") if isinstance(item, dict) else str(item) + ) if not value: continue embed_text = f"{label}: {value}" @@ -133,7 +137,9 @@ def _make_entity(embed_text: str, item_type: str) -> Dict[str, Any]: # user_goal (single string) user_goal = profile_data.get("user_goal") if user_goal and isinstance(user_goal, str) and user_goal.strip(): - entities.append(_make_entity(f"Goal: {user_goal.strip()}", "explicit_info")) + entities.append( + _make_entity(f"Goal: {user_goal.strip()}", "explicit_info") + ) return entities diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/repository/atomic_fact_es_repository.py b/methods/EverCore/src/infra_layer/adapters/out/search/repository/atomic_fact_es_repository.py index ba98434e..fe37dac6 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/repository/atomic_fact_es_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/repository/atomic_fact_es_repository.py @@ -183,10 +183,14 @@ async def multi_search( else: # user_id must not exist: match docs where field is missing or "" filter_queries.append( - Q("bool", should=[ - Q("bool", must_not=[Q("exists", field="user_id")]), - Q("term", user_id=""), - ], minimum_should_match=1) + Q( + "bool", + should=[ + Q("bool", must_not=[Q("exists", field="user_id")]), + Q("term", user_id=""), + ], + minimum_should_match=1, + ) ) # Handle group_ids filter diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/repository/episodic_memory_es_repository.py b/methods/EverCore/src/infra_layer/adapters/out/search/repository/episodic_memory_es_repository.py index bf38e6ef..14e326ac 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/repository/episodic_memory_es_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/repository/episodic_memory_es_repository.py @@ -196,10 +196,14 @@ async def multi_search( else: # user_id must not exist: match docs where field is missing or "" filter_queries.append( - Q("bool", should=[ - Q("bool", must_not=[Q("exists", field="user_id")]), - Q("term", user_id=""), - ], minimum_should_match=1) + Q( + "bool", + should=[ + Q("bool", must_not=[Q("exists", field="user_id")]), + Q("term", user_id=""), + ], + minimum_should_match=1, + ) ) # Handle group_ids filter diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/repository/foresight_es_repository.py b/methods/EverCore/src/infra_layer/adapters/out/search/repository/foresight_es_repository.py index ed15a6b5..0ebfb30d 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/repository/foresight_es_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/repository/foresight_es_repository.py @@ -213,10 +213,14 @@ async def multi_search( else: # user_id must not exist: match docs where field is missing or "" filter_queries.append( - Q("bool", should=[ - Q("bool", must_not=[Q("exists", field="user_id")]), - Q("term", user_id=""), - ], minimum_should_match=1) + Q( + "bool", + should=[ + Q("bool", must_not=[Q("exists", field="user_id")]), + Q("term", user_id=""), + ], + minimum_should_match=1, + ) ) # Handle group_ids filter diff --git a/methods/EverCore/src/manage.py b/methods/EverCore/src/manage.py index eac4c004..db8cf00c 100644 --- a/methods/EverCore/src/manage.py +++ b/methods/EverCore/src/manage.py @@ -202,7 +202,7 @@ def list_commands( typer.echo("Available commands:") for cmd in commands: help_text = cmd.help if cmd.help else "No description" - typer.echo(f" {cmd.name:<20} {help_text}"), + (typer.echo(f" {cmd.name:<20} {help_text}"),) typer.echo(f"\nUsing environment file: {env_file}") @@ -211,7 +211,7 @@ def list_commands( def tenant_init( env_file: str = typer.Option( ".env", "--env-file", help="Specify the environment variable file to load" - ) + ), ): """ Initialize MongoDB and Milvus databases for a specific tenant diff --git a/methods/EverCore/src/memory_layer/cluster_manager/manager.py b/methods/EverCore/src/memory_layer/cluster_manager/manager.py index 307ff2da..81da6f70 100644 --- a/methods/EverCore/src/memory_layer/cluster_manager/manager.py +++ b/methods/EverCore/src/memory_layer/cluster_manager/manager.py @@ -30,7 +30,6 @@ logger.warning("Vectorize service not available, clustering will be limited") - class MemSceneState: """Internal state for a single group's clustering.""" @@ -260,10 +259,7 @@ def on_cluster_assigned( self._callbacks.append(callback) async def cluster_memcell( - self, - memcell: Dict[str, Any], - state: MemSceneState, - has_case: bool = False, + self, memcell: Dict[str, Any], state: MemSceneState, has_case: bool = False ) -> Tuple[Optional[str], MemSceneState]: """Cluster a memcell and return updated state. @@ -382,9 +378,7 @@ def _assign_to_cluster( ) if timestamp is not None: prev_ts = state.cluster_last_ts.get(cluster_id) - state.cluster_last_ts[cluster_id] = max( - prev_ts or timestamp, timestamp - ) + state.cluster_last_ts[cluster_id] = max(prev_ts or timestamp, timestamp) def _append_event( self, @@ -401,9 +395,7 @@ def _append_event( ) async def _cluster_memcell_llm( - self, - memcell: Dict[str, Any], - state: MemSceneState, + self, memcell: Dict[str, Any], state: MemSceneState ) -> Tuple[Optional[str], MemSceneState]: """LLM-based clustering with embedding pre-filtering. @@ -428,13 +420,11 @@ async def _cluster_memcell_llm( ) vector = await self._get_embedding(text) best_cid = self._find_top_k_clusters( - state, vector, k=1, only_cids=state.case_cluster_ids, + state, vector, k=1, only_cids=state.case_cluster_ids ) if best_cid and best_cid[0][1] >= self.config.similarity_threshold: cluster_id = best_cid[0][0] - self._assign_to_cluster( - state, event_id, cluster_id, vector, timestamp - ) + self._assign_to_cluster(state, event_id, cluster_id, vector, timestamp) else: cluster_id = self._create_new_cluster( state, event_id, vector, timestamp, is_case=True @@ -459,7 +449,8 @@ async def _cluster_memcell_llm( # Stage 1: Embedding recall — find top-K candidate clusters (case only) vector = await self._get_embedding(text) scored_candidates = self._find_top_k_clusters( - state, vector, + state, + vector, k=self.config.llm_top_k_clusters, only_cids=state.case_cluster_ids, ) @@ -487,17 +478,13 @@ async def _cluster_memcell_llm( cluster_context = await self._fetch_cluster_context(state, candidate_ids) # Stage 3: LLM decision - clusters_json = self._build_clusters_json( - state, candidate_ids, cluster_context - ) + clusters_json = self._build_clusters_json(state, candidate_ids, cluster_context) next_new_id = f"{state.next_cluster_idx:03d}" from memory_layer.prompts import get_prompt_by prompt_template = get_prompt_by("AGENT_CLUSTER_LLM_ASSIGN_PROMPT") prompt = prompt_template.format( - memcell_text=text, - clusters_json=clusters_json, - next_new_id=next_new_id, + memcell_text=text, clusters_json=clusters_json, next_new_id=next_new_id ) llm_result = await self._call_llm_for_clustering(prompt) @@ -507,22 +494,24 @@ async def _cluster_memcell_llm( f"falling back to embedding top-1" ) # Fall back to embedding: use top-1 candidate if available, else new cluster - if scored_candidates and scored_candidates[0][1] >= self.config.similarity_threshold: + if ( + scored_candidates + and scored_candidates[0][1] >= self.config.similarity_threshold + ): cluster_id = scored_candidates[0][0] - self._assign_to_cluster( - state, event_id, cluster_id, vector, timestamp - ) + self._assign_to_cluster(state, event_id, cluster_id, vector, timestamp) else: cluster_id = self._create_new_cluster( state, event_id, vector, timestamp, is_case=True ) else: chosen_id = llm_result.get("cluster_id", "") - if chosen_id in state.cluster_counts and chosen_id in state.case_cluster_ids: + if ( + chosen_id in state.cluster_counts + and chosen_id in state.case_cluster_ids + ): cluster_id = chosen_id - self._assign_to_cluster( - state, event_id, cluster_id, vector, timestamp - ) + self._assign_to_cluster(state, event_id, cluster_id, vector, timestamp) else: cluster_id = self._create_new_cluster( state, event_id, vector, timestamp, is_case=True @@ -579,9 +568,7 @@ def _find_top_k_clusters( return scored[:k] async def _fetch_cluster_context( - self, - state: MemSceneState, - candidate_ids: List[str], + self, state: MemSceneState, candidate_ids: List[str] ) -> Dict[str, List[str]]: """Fetch recent context texts for candidate clusters via context_fetcher. @@ -640,17 +627,11 @@ def _build_clusters_json( count = state.cluster_counts.get(cid, 0) recent = cluster_context.get(cid, []) clusters.append( - { - "cluster_id": cid, - "item_count": count, - "recent_task_intents": recent, - } + {"cluster_id": cid, "item_count": count, "recent_task_intents": recent} ) return json.dumps(clusters, ensure_ascii=False, indent=2) - async def _call_llm_for_clustering( - self, prompt: str - ) -> Optional[Dict[str, Any]]: + async def _call_llm_for_clustering(self, prompt: str) -> Optional[Dict[str, Any]]: """Call LLM and parse clustering decision.""" for attempt in range(3): try: @@ -664,9 +645,7 @@ async def _call_llm_for_clustering( f"[LLM Clustering] Retry {attempt + 1}/3: invalid response format" ) except Exception as e: - logger.warning( - f"[LLM Clustering] Retry {attempt + 1}/3: {e}" - ) + logger.warning(f"[LLM Clustering] Retry {attempt + 1}/3: {e}") return None def _find_best_cluster( diff --git a/methods/EverCore/src/memory_layer/llm/__init__.py b/methods/EverCore/src/memory_layer/llm/__init__.py index 03d92194..a071f5ec 100644 --- a/methods/EverCore/src/memory_layer/llm/__init__.py +++ b/methods/EverCore/src/memory_layer/llm/__init__.py @@ -31,10 +31,7 @@ def create_provider(provider_type: str, **kwargs) -> LLMProvider: ) return OpenAIProvider( - provider_type=provider_type, - api_key=api_key, - base_url=base_url, - **kwargs, + provider_type=provider_type, api_key=api_key, base_url=base_url, **kwargs ) @@ -58,8 +55,5 @@ def create_provider_from_env(provider_type: str, **kwargs) -> LLMProvider: ) return OpenAIProvider( - provider_type=provider_type, - api_key=api_key, - base_url=base_url, - **kwargs, + provider_type=provider_type, api_key=api_key, base_url=base_url, **kwargs ) diff --git a/methods/EverCore/src/memory_layer/llm/config.py b/methods/EverCore/src/memory_layer/llm/config.py index e6a616eb..1343e58a 100644 --- a/methods/EverCore/src/memory_layer/llm/config.py +++ b/methods/EverCore/src/memory_layer/llm/config.py @@ -31,7 +31,9 @@ def create_provider( Returns: Configured OpenAIProvider instance """ - api_key, base_url = resolve_provider_env("openai", api_key=api_key, base_url=base_url) + api_key, base_url = resolve_provider_env( + "openai", api_key=api_key, base_url=base_url + ) return OpenAIProvider( model=model, api_key=api_key, diff --git a/methods/EverCore/src/memory_layer/memcell_extractor/agent_memcell_extractor.py b/methods/EverCore/src/memory_layer/memcell_extractor/agent_memcell_extractor.py index 8bcc1f94..642c7b7c 100644 --- a/methods/EverCore/src/memory_layer/memcell_extractor/agent_memcell_extractor.py +++ b/methods/EverCore/src/memory_layer/memcell_extractor/agent_memcell_extractor.py @@ -126,22 +126,21 @@ async def extract_memcell( if request.new_raw_data_list: # Skip when new messages are all user messages (no assistant response yet) all_user_only = all( - isinstance(rd.content, dict) - and rd.content.get("role") == "user" + isinstance(rd.content, dict) and rd.content.get("role") == "user" for rd in request.new_raw_data_list ) if all_user_only: logger.debug( "[AgentMemCellExtractor] Skipping: new messages contain " - "only user messages, waiting for assistant response", + "only user messages, waiting for assistant response" ) return ([], StatusResult(should_wait=True)) # Skip when the agent turn is still in progress last_content = request.new_raw_data_list[-1].content - if isinstance( - last_content, dict - ) and is_intermediate_agent_step(last_content): + if isinstance(last_content, dict) and is_intermediate_agent_step( + last_content + ): logger.debug( "[AgentMemCellExtractor] Skipping: last new message is " "intermediate (role=%s)", @@ -167,10 +166,7 @@ def _is_safe_split(messages: List[Dict[str, Any]], split_at: int) -> bool: if split_at < 1 or split_at > len(messages) - 1: return False last_msg = messages[split_at - 1] - return ( - last_msg.get("role") == "assistant" - and not last_msg.get("tool_calls") - ) + return last_msg.get("role") == "assistant" and not last_msg.get("tool_calls") def _find_force_split_point(self, messages: List[Dict[str, Any]]) -> int: """Find force-split point that does not break a tool-call sequence. diff --git a/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py b/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py index 56b91b29..424dc34e 100644 --- a/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py +++ b/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py @@ -182,7 +182,7 @@ def _find_force_split_point(self, messages: List[Dict[str, Any]]) -> int: @staticmethod def _build_original_data_items( - messages: List[Dict[str, Any]] + messages: List[Dict[str, Any]], ) -> List[Dict[str, Any]]: """Build original_data items in { message } format. @@ -410,7 +410,7 @@ async def _detect_boundaries( for i in range(5): resp = await self.llm_provider.generate(prompt) logger.debug( - f"[ConvMemCellExtractor] === BOUNDARY DETECTION RESPONSE (attempt {i+1}) ===\n" + f"[ConvMemCellExtractor] === BOUNDARY DETECTION RESPONSE (attempt {i + 1}) ===\n" f"{resp}\n" f"[ConvMemCellExtractor] === END RESPONSE ===" ) diff --git a/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py index 15afd267..aaf49636 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py @@ -32,11 +32,7 @@ MemoryExtractRequest, ) from memory_layer.prompts import get_prompt_by -from api_specs.memory_types import ( - RawDataType, - AgentCase, - get_text_from_content_items, -) +from api_specs.memory_types import RawDataType, AgentCase, get_text_from_content_items from api_specs.memory_models import MemoryType from agentic_layer.vectorize_service import get_vectorize_service from core.di.utils import get_bean_by_type @@ -281,7 +277,7 @@ async def _pre_compress_to_list( groups_by_size = sorted(groups_with_size, key=lambda x: x[2], reverse=True) compress_indices: set = set() estimated_total = total_size - for idx, group, size in groups_by_size: + for idx, _group, size in groups_by_size: if estimated_total <= self.pre_compress_chunk_size: break compress_indices.add(idx) @@ -384,12 +380,12 @@ async def _compress_tool_chunk( ): return data["compressed_messages"] logger.warning( - f"[AgentCaseExtractor] Tool pre-compress attempt {attempt+1}/2: " + f"[AgentCaseExtractor] Tool pre-compress attempt {attempt + 1}/2: " f"invalid response format" ) except Exception as e: logger.warning( - f"[AgentCaseExtractor] Tool pre-compress attempt {attempt+1}/2: {e}" + f"[AgentCaseExtractor] Tool pre-compress attempt {attempt + 1}/2: {e}" ) return None @@ -434,12 +430,12 @@ async def _compress_experience( return None return data logger.warning( - f"[AgentCaseExtractor] Compress attempt {attempt+1}/2: " + f"[AgentCaseExtractor] Compress attempt {attempt + 1}/2: " f"missing or invalid 'task_intent' field" ) except Exception as e: logger.warning( - f"[AgentCaseExtractor] Compress attempt {attempt+1}/2: {e}" + f"[AgentCaseExtractor] Compress attempt {attempt + 1}/2: {e}" ) logger.error( @@ -499,7 +495,7 @@ def _count_tool_call_rounds(messages: List[Dict[str, Any]]) -> int: @staticmethod def _strip_before_first_user( - messages: List[Dict[str, Any]] + messages: List[Dict[str, Any]], ) -> List[Dict[str, Any]]: """Drop messages before the first user message (e.g. system prompts).""" for i, msg in enumerate(messages): @@ -612,7 +608,9 @@ async def extract_memory( # with limits inversely proportional to how far over the threshold we are. # After trim, skip entirely if still over 2x PRE_COMPRESS_CHUNK_SIZE. total_tokens = self._count_tokens( - json.dumps(original_data, ensure_ascii=False, default=self._json_default) + json.dumps( + original_data, ensure_ascii=False, default=self._json_default + ) ) logger.info( f"[AgentCaseExtractor] event_id={memcell.event_id}, " @@ -633,7 +631,9 @@ async def extract_memory( scale = scale_trigger / total_tokens trim_tool_output = max(200, int(self.max_tool_output_tokens * scale)) trim_tool_args = max(200, int(self.max_tool_args_tokens * scale)) - trim_assistant = max(500, int(self.max_assistant_response_tokens * scale)) + trim_assistant = max( + 500, int(self.max_assistant_response_tokens * scale) + ) logger.info( f"[AgentCaseExtractor] Total tokens {total_tokens} > " f"scale_trigger ({scale_trigger}), " @@ -646,17 +646,16 @@ async def extract_memory( trim_tool_args = self.max_tool_args_tokens trim_assistant = self.max_assistant_response_tokens original_data = self._heuristic_trim_tool_outputs( - original_data, - trim_tool_output, - trim_tool_args, - trim_assistant, + original_data, trim_tool_output, trim_tool_args, trim_assistant ) # Only re-count after trim when scaling was applied — if total_tokens was # already <= scale_trigger, trimmed_tokens can't possibly exceed 2x chunk_size. if needs_scale: trimmed_tokens = self._count_tokens( - json.dumps(original_data, ensure_ascii=False, default=self._json_default) + json.dumps( + original_data, ensure_ascii=False, default=self._json_default + ) ) if trimmed_tokens > self.pre_compress_chunk_size * 2: logger.info( diff --git a/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py index 121b7320..3895fc37 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py @@ -122,10 +122,7 @@ def _get_tokenizer(cls): @classmethod def _truncate_text( - cls, - text: str, - max_tokens: int = 200, - suffix: str = "... [omitted]", + cls, text: str, max_tokens: int = 200, suffix: str = "... [omitted]" ) -> str: """Truncate text to max_tokens using tokenizer, appending suffix if truncated.""" if not text or not isinstance(text, str): @@ -151,7 +148,9 @@ def _summarize_case_for_prompt( entry["key_insight"] = key_insight approach = getattr(case_record, "approach", None) if approach: - entry["approach"] = self._truncate_text(approach, max_tokens=max_approach_tokens) + entry["approach"] = self._truncate_text( + approach, max_tokens=max_approach_tokens + ) return entry def _format_existing_skills( @@ -171,7 +170,7 @@ def _format_existing_skills( # Build lookup: case_id -> case_record case_map: Dict[str, Any] = {} - for rec in (case_history or []): + for rec in case_history or []: cid = str(getattr(rec, "id", "") or "") if cid: case_map[cid] = rec @@ -181,8 +180,12 @@ def _format_existing_skills( item: Dict[str, Any] = { "index": idx, "name": rec.name, - "description": self._truncate_text(rec.description, max_tokens=self.MAX_DESCRIPTION_TOKENS), - "content": self._truncate_text(rec.content, max_tokens=self.MAX_CONTENT_TOKENS), + "description": self._truncate_text( + rec.description, max_tokens=self.MAX_DESCRIPTION_TOKENS + ), + "content": self._truncate_text( + rec.content, max_tokens=self.MAX_CONTENT_TOKENS + ), "confidence": rec.confidence, } @@ -200,7 +203,9 @@ def _format_existing_skills( for sid in recent_ids ] - lines.append(json.dumps(item, ensure_ascii=False, default=self._json_default)) + lines.append( + json.dumps(item, ensure_ascii=False, default=self._json_default) + ) return "[\n" + ",\n".join(lines) + "\n]" @staticmethod @@ -339,7 +344,9 @@ async def _evaluate_maturity( then normalizes to 0.0-1.0. """ if self.skip_maturity_scoring: - logger.info("[AgentSkillExtractor] Maturity scoring skipped by config, returning 1.0") + logger.info( + "[AgentSkillExtractor] Maturity scoring skipped by config, returning 1.0" + ) return 1.0 try: prompt = self.maturity_prompt.format( @@ -418,7 +425,7 @@ def _is_skill_content_sufficient( stripped = content.strip() if len(stripped) < min_length: return False - non_empty_lines = [l for l in stripped.splitlines() if l.strip()] + non_empty_lines = [line for line in stripped.splitlines() if line.strip()] return len(non_empty_lines) >= min_lines async def _apply_add( @@ -454,7 +461,9 @@ async def _apply_add( "[AgentSkillExtractor] add operation has no name and no description, skipping" ) return None - description = self._truncate_text(description, max_tokens=self.MAX_DESCRIPTION_TOKENS, suffix="...") + description = self._truncate_text( + description, max_tokens=self.MAX_DESCRIPTION_TOKENS, suffix="..." + ) try: confidence = max(0.0, min(1.0, float(data.get("confidence", 0.5)))) @@ -545,7 +554,9 @@ async def _apply_update( new_name = data.get("name", "") new_description = data.get("description", "") - new_description = self._truncate_text(new_description, max_tokens=self.MAX_DESCRIPTION_TOKENS, suffix="...") + new_description = self._truncate_text( + new_description, max_tokens=self.MAX_DESCRIPTION_TOKENS, suffix="..." + ) new_content = data.get("content", "") new_confidence = data.get("confidence") @@ -604,9 +615,7 @@ async def _apply_update( retire_updates: Dict[str, Any] = {"confidence": final_confidence} if "source_case_ids" in updates: retire_updates["source_case_ids"] = updates["source_case_ids"] - success = await skill_repo.update_skill_by_id( - record_id, retire_updates - ) + success = await skill_repo.update_skill_by_id(record_id, retire_updates) if success: # Signal search-engine removal (ES / Milvus) — data stays in MongoDB result.deleted_ids.append(str(record_id)) @@ -736,9 +745,7 @@ async def _apply_update( return success async def _load_case_history( - self, - existing_skill_records: List[Any], - max_cases: int = 9, + self, existing_skill_records: List[Any], max_cases: int = 9 ) -> List[Any]: """Load historical AgentCaseRecords referenced by existing skills. @@ -747,7 +754,7 @@ async def _load_case_history( """ all_case_ids: set = set() for rec in existing_skill_records: - for cid in (getattr(rec, "source_case_ids", None) or []): + for cid in getattr(rec, "source_case_ids", None) or []: if cid is None: continue cid_str = str(cid).strip() @@ -771,7 +778,9 @@ async def _load_case_history( ) logger.info( "[AgentSkillExtractor] Loaded case_history: %d/%d cases (max=%d)", - min(len(records), max_cases), len(records), max_cases, + min(len(records), max_cases), + len(records), + max_cases, ) return records[:max_cases] except Exception as e: @@ -856,8 +865,7 @@ async def extract_and_save( # Collect all case IDs from new records for traceability source_case_ids = [ - str(getattr(rec, "id", "") or "") - for rec in new_case_records + str(getattr(rec, "id", "") or "") for rec in new_case_records ] source_case_ids = [cid for cid in source_case_ids if cid] diff --git a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py index 86980b7f..b223ac53 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py @@ -268,7 +268,9 @@ async def extract_atomic_fact( input_text, timestamp, user_id=user_id, group_id=group_id ) except Exception as e: - logger.warning(f"Retrying to extract atomic fact {retry+1}/5: {e}") + logger.warning( + f"Retrying to extract atomic fact {retry + 1}/5: {e}" + ) if retry == 4: logger.error("Failed to extract atomic fact after 5 retries") return None diff --git a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py index 4b6985bd..194ef8d9 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py @@ -297,7 +297,7 @@ async def _extract_episode( # Validation passed, exit retry loop break except Exception as e: - logger.warning(f"Episode extraction retry {i+1}/5: {e}") + logger.warning(f"Episode extraction retry {i + 1}/5: {e}") if i == 4: raise Exception( "Episode memory extraction failed after 5 retries" diff --git a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py index c05fb8ea..9b41e09c 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py @@ -154,7 +154,7 @@ async def generate_foresights_for_conversation( return foresights except Exception as e: - logger.warning(f"Foresight generation retry {retry+1}/5: {e}") + logger.warning(f"Foresight generation retry {retry + 1}/5: {e}") if retry == 4: logger.error("Foresight generation failed after 5 retries") return [] diff --git a/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py index 2981093e..8a0f0fd2 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py @@ -24,7 +24,14 @@ MemoryExtractRequest, ) from memory_layer.prompts import get_prompt_by -from api_specs.memory_types import MemCell, MemoryType, ProfileMemory, ScenarioType, get_text_from_content_items, is_intermediate_agent_step +from api_specs.memory_types import ( + MemCell, + MemoryType, + ProfileMemory, + ScenarioType, + get_text_from_content_items, + is_intermediate_agent_step, +) logger = get_logger(__name__) @@ -33,8 +40,9 @@ # ID Mapper — Long ID <-> Short ID conversion to save tokens # ============================================================================ + def _create_id_mapping(long_ids: List[str]) -> Dict[str, str]: - return {lid: f"ep{i+1}" for i, lid in enumerate(long_ids) if lid} + return {lid: f"ep{i + 1}" for i, lid in enumerate(long_ids) if lid} def _replace_sources( @@ -68,6 +76,7 @@ def _get_short_id(long_id: str, id_map: Dict[str, str]) -> str: # Extract Request # ============================================================================ + class ProfileAction(str, Enum): NONE = "none" ADD = "add" @@ -116,6 +125,7 @@ def __post_init__(self): # Profile Extractor # ============================================================================ + class ProfileExtractor(MemoryExtractor): """Extracts user profiles using incremental operations (add/update/delete).""" @@ -147,7 +157,9 @@ async def extract_memory( # Initialize profile if old_profile is None: - logger.info(f"[ProfileExtractor] No old_profile for user={request.user_id}, creating new") + logger.info( + f"[ProfileExtractor] No old_profile for user={request.user_id}, creating new" + ) current_profile = ProfileMemory( memory_type=MemoryType.PROFILE, user_id=request.user_id or "", @@ -196,11 +208,13 @@ async def extract_memory( if updated_dict: current_profile.explicit_info = [ - d for d in updated_dict.get(ProfileItemType.EXPLICIT_INFO, []) + d + for d in updated_dict.get(ProfileItemType.EXPLICIT_INFO, []) if d.get("description", "").strip() ] current_profile.implicit_traits = [ - d for d in updated_dict.get(ProfileItemType.IMPLICIT_TRAITS, []) + d + for d in updated_dict.get(ProfileItemType.IMPLICIT_TRAITS, []) if d.get("description", "").strip() ] current_profile.last_updated = get_now_with_timezone() @@ -291,25 +305,35 @@ async def _llm_update_profile( ] if op_type == ProfileItemType.EXPLICIT_INFO: explicit_list.append(data) - logger.info(f"[Profile] Added explicit_info: {data.get('description', '')[:30]}...") + logger.info( + f"[Profile] Added explicit_info: {data.get('description', '')[:30]}..." + ) elif op_type == ProfileItemType.IMPLICIT_TRAITS: implicit_list.append(data) - logger.info(f"[Profile] Added implicit_trait: {data.get('trait', '')}...") + logger.info( + f"[Profile] Added implicit_trait: {data.get('trait', '')}..." + ) elif action == ProfileAction.UPDATE: op_type = op.get("type") index = op.get("index", -1) data = op.get("data", {}) target_list = ( - explicit_list if op_type == ProfileItemType.EXPLICIT_INFO else implicit_list + explicit_list + if op_type == ProfileItemType.EXPLICIT_INFO + else implicit_list ) if 0 <= index < len(target_list): for key, val in data.items(): if val: if key == "sources": old_sources = target_list[index].get("sources", []) - new_sources = [self._attach_ts(s, id_to_ts) for s in val] - target_list[index]["sources"] = list(set(old_sources + new_sources)) + new_sources = [ + self._attach_ts(s, id_to_ts) for s in val + ] + target_list[index]["sources"] = list( + set(old_sources + new_sources) + ) else: target_list[index][key] = val logger.info(f"[Profile] Updated {op_type}[{index}]") @@ -319,11 +343,15 @@ async def _llm_update_profile( index = op.get("index", -1) reason = op.get("reason", "") target_list = ( - explicit_list if op_type == ProfileItemType.EXPLICIT_INFO else implicit_list + explicit_list + if op_type == ProfileItemType.EXPLICIT_INFO + else implicit_list ) if 0 <= index < len(target_list) and reason: target_list.pop(index) - logger.warning(f"[Profile] Deleted {op_type}[{index}]: {reason}") + logger.warning( + f"[Profile] Deleted {op_type}[{index}]: {reason}" + ) result_dict = { ProfileItemType.EXPLICIT_INFO: explicit_list, @@ -377,14 +405,18 @@ def _format_profile_with_index(self, profile_dict: Dict[str, Any]) -> str: if explicit: lines.append("【Explicit Info】") for i, item in enumerate(explicit): - lines.append(f" [{i}] [{item.get('category', '')}] {item.get('description', '')}") + lines.append( + f" [{i}] [{item.get('category', '')}] {item.get('description', '')}" + ) if item.get("evidence"): lines.append(f" evidence: {item['evidence']}") if implicit: lines.append("\n【Implicit Traits】") for i, item in enumerate(implicit): - lines.append(f" [{i}] {item.get('trait', '')}: {item.get('description', '')}") + lines.append( + f" [{i}] {item.get('trait', '')}: {item.get('description', '')}" + ) if item.get("evidence"): lines.append(f" evidence: {item['evidence']}") @@ -410,11 +442,13 @@ async def _compact_profile( if result: result_long = _replace_sources(result, id_map, reverse=True) profile.explicit_info = [ - d for d in result_long.get(ProfileItemType.EXPLICIT_INFO, []) + d + for d in result_long.get(ProfileItemType.EXPLICIT_INFO, []) if d.get("description", "").strip() ] profile.implicit_traits = [ - d for d in result_long.get(ProfileItemType.IMPLICIT_TRAITS, []) + d + for d in result_long.get(ProfileItemType.IMPLICIT_TRAITS, []) if d.get("description", "").strip() ] profile.last_updated = get_now_with_timezone() @@ -436,7 +470,9 @@ def _format_profile_for_llm(self, profile_dict: Dict[str, Any]) -> str: if explicit: lines.append("【Explicit Info】") for i, item in enumerate(explicit, 1): - lines.append(f" {i}. [{item.get('category', '')}] {item.get('description', '')}") + lines.append( + f" {i}. [{item.get('category', '')}] {item.get('description', '')}" + ) if item.get("evidence"): lines.append(f" evidence: {item['evidence']}") lines.append(f" sources: {', '.join(item.get('sources', []))}") @@ -444,7 +480,9 @@ def _format_profile_for_llm(self, profile_dict: Dict[str, Any]) -> str: if implicit: lines.append("\n【Implicit Traits】") for i, item in enumerate(implicit, 1): - lines.append(f" {i}. {item.get('trait', '')}: {item.get('description', '')}") + lines.append( + f" {i}. {item.get('trait', '')}: {item.get('description', '')}" + ) if item.get("basis"): lines.append(f" basis: {item['basis']}") if item.get("evidence"): @@ -506,7 +544,9 @@ def _resolve_user_name( if name: return name # Fallback to user_id itself - logger.warning(f"Could not resolve sender_name for user_id={user_id}, using user_id as fallback") + logger.warning( + f"Could not resolve sender_name for user_id={user_id}, using user_id as fallback" + ) return user_id def _parse_profile_response(self, response: str) -> Optional[Dict[str, Any]]: diff --git a/methods/EverCore/src/memory_layer/memory_manager.py b/methods/EverCore/src/memory_layer/memory_manager.py index 01a26316..4c97a71a 100644 --- a/methods/EverCore/src/memory_layer/memory_manager.py +++ b/methods/EverCore/src/memory_layer/memory_manager.py @@ -199,6 +199,7 @@ async def extract_memcell( from memory_layer.memcell_extractor.agent_memcell_extractor import ( AgentMemCellExtractor, ) + extractor = AgentMemCellExtractor(self._get_provider_for_scene("boundary")) else: extractor = ConvMemCellExtractor(self._get_provider_for_scene("boundary")) @@ -376,10 +377,8 @@ async def _extract_foresight( if uid is None: display_name = ",".join( { - - item.get("message", item).get("sender_name") - for item in memcell.original_data or [] - + item.get("message", item).get("sender_name") + for item in memcell.original_data or [] } ) else: @@ -463,8 +462,6 @@ async def _extract_agent_case( llm_provider=self._get_provider_for_scene("extraction") ) request = AgentCaseExtractRequest( - memcell=memcell, - user_id=user_id, - group_id=group_id, + memcell=memcell, user_id=user_id, group_id=group_id ) return await extractor.extract_memory(request) diff --git a/methods/EverCore/src/memory_layer/profile_indexer/__init__.py b/methods/EverCore/src/memory_layer/profile_indexer/__init__.py index 637d7495..872f8665 100644 --- a/methods/EverCore/src/memory_layer/profile_indexer/__init__.py +++ b/methods/EverCore/src/memory_layer/profile_indexer/__init__.py @@ -6,7 +6,4 @@ from .profile_indexer import ProfileIndexer, index_user_profile -__all__ = [ - "ProfileIndexer", - "index_user_profile", -] +__all__ = ["ProfileIndexer", "index_user_profile"] diff --git a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py index ebc8948e..35987792 100644 --- a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py +++ b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py @@ -155,7 +155,9 @@ async def index_profile( if not source_doc or not source_doc.id: logger.error( "[ProfileIndexer] Cannot find MongoDB profile doc: user_id=%s, group_id=%s, doc_id=%s", - user_id, group_id, doc_id, + user_id, + group_id, + doc_id, ) return stats diff --git a/methods/EverCore/src/memory_layer/profile_manager/manager.py b/methods/EverCore/src/memory_layer/profile_manager/manager.py index fdf05120..6e6bf465 100644 --- a/methods/EverCore/src/memory_layer/profile_manager/manager.py +++ b/methods/EverCore/src/memory_layer/profile_manager/manager.py @@ -158,7 +158,8 @@ async def extract_profiles( user_baseline = new_context.get("created_at") user_cluster_episodes = [ - ep for ep in cluster_contexts + ep + for ep in cluster_contexts if ep.get("created_at") is None or ep.get("created_at") > user_baseline ] diff --git a/methods/EverCore/src/memory_layer/prompts/__init__.py b/methods/EverCore/src/memory_layer/prompts/__init__.py index 453b1a14..734f210b 100644 --- a/methods/EverCore/src/memory_layer/prompts/__init__.py +++ b/methods/EverCore/src/memory_layer/prompts/__init__.py @@ -6,7 +6,7 @@ Example: from memory_layer.prompts import get_prompt_by - + prompt = get_prompt_by("EPISODE_GENERATION_PROMPT") # default language prompt = get_prompt_by("EPISODE_GENERATION_PROMPT", language="zh") # specific language """ @@ -113,11 +113,6 @@ "en": ("memory_layer.prompts.en.agent_prompts", False), "zh": ("memory_layer.prompts.zh.agent_prompts", False), }, - # Skill relevance verification - "AGENT_SKILL_RELEVANCE_VERIFY_PROMPT": { - "en": ("memory_layer.prompts.en.agent_prompts", False), - "zh": ("memory_layer.prompts.zh.agent_prompts", False), - }, } diff --git a/methods/EverCore/src/memory_layer/prompts/en/atomic_fact_prompts.py b/methods/EverCore/src/memory_layer/prompts/en/atomic_fact_prompts.py index 2a2e203b..6db94b48 100644 --- a/methods/EverCore/src/memory_layer/prompts/en/atomic_fact_prompts.py +++ b/methods/EverCore/src/memory_layer/prompts/en/atomic_fact_prompts.py @@ -6,7 +6,7 @@ ATOMIC_FACT_PROMPT = """ **CRITICAL LANGUAGE RULE**: You MUST output in the SAME language as the input conversation content. If the conversation content is in Chinese, ALL output MUST be in Chinese. If in English, output in English. This is mandatory. -You are an expert information extraction analyst and information architect. +You are an expert information extraction analyst and information architect. Your task is to analyze the given raw conversation transcript (called "CONVERSATION_TEXT") and produce atomic facts optimized for factual retrieval. --- diff --git a/methods/EverCore/src/memory_layer/prompts/zh/conv_prompts.py b/methods/EverCore/src/memory_layer/prompts/zh/conv_prompts.py index f1cebca6..11c60675 100644 --- a/methods/EverCore/src/memory_layer/prompts/zh/conv_prompts.py +++ b/methods/EverCore/src/memory_layer/prompts/zh/conv_prompts.py @@ -26,7 +26,7 @@ - **主题切换:** 对话从“讨论项目A的技术细节”突然转向“周末去哪里玩”。 - **任务完成并开启新篇:** 一个任务的收尾消息(如“仓库迁移好了”)属于该任务的情节。只有当**下一条**消息开启了完全不相关的新话题时,才在那条新消息处切分。 - **长时间中断后开启新话题:** 时间间隔超过4小时,且新消息内容与历史对话无明显关联。 - + 2. **`should_wait` (等待更多信息):** - **何时设为 `true`?** 当新消息**信息量不足**,无法判断其是否延续当前话题时。这是一个**安全选项**,用于避免在上下文不充分时草率地分割或合并对话。 diff --git a/methods/EverCore/src/memory_layer/prompts/zh/group_profile_merge_prompts.py b/methods/EverCore/src/memory_layer/prompts/zh/group_profile_merge_prompts.py index 162259ab..2266de7f 100644 --- a/methods/EverCore/src/memory_layer/prompts/zh/group_profile_merge_prompts.py +++ b/methods/EverCore/src/memory_layer/prompts/zh/group_profile_merge_prompts.py @@ -182,4 +182,3 @@ - **示例**:如果原始内容是中文,user_goal、subtasks 应该是中文,但 personality 应该保持为"Extraversion/NeedForBelonging/等"。 """ - diff --git a/methods/EverCore/src/run.py b/methods/EverCore/src/run.py index 9d5aced7..0981848f 100644 --- a/methods/EverCore/src/run.py +++ b/methods/EverCore/src/run.py @@ -8,6 +8,7 @@ - Full-text writing and editing agent - Document management and resource processing services """ + import argparse import os import sys diff --git a/methods/EverCore/src/service/memcell_delete_service.py b/methods/EverCore/src/service/memcell_delete_service.py index c3b4e02d..6ac8a6a1 100644 --- a/methods/EverCore/src/service/memcell_delete_service.py +++ b/methods/EverCore/src/service/memcell_delete_service.py @@ -156,8 +156,7 @@ async def delete_by_filters( MemCell itself is not deleted (only child records). """ logger.info( - "Deleting by filters: user_id=%s, group_id=%s, " - "session_id=%s, sender_id=%s", + "Deleting by filters: user_id=%s, group_id=%s, session_id=%s, sender_id=%s", user_id, group_id, session_id, diff --git a/methods/EverCore/src/service/settings_service.py b/methods/EverCore/src/service/settings_service.py index 2f5908f5..b1a0e080 100644 --- a/methods/EverCore/src/service/settings_service.py +++ b/methods/EverCore/src/service/settings_service.py @@ -119,7 +119,6 @@ async def update( logger.info("Settings initialized") return self._to_response(doc) else: - update_data = self._build_update_data(request, raw_data=raw_data) if not update_data: # No fields to update, return current state @@ -237,7 +236,7 @@ def _build_data( """Build data dict from request for initialization""" data = {} fields = [ - "llm_custom_setting", + "llm_custom_setting" # Hidden fields: not yet implemented, uncomment when ready # "timezone", # "boundary_detection_timeout", @@ -265,7 +264,7 @@ def _build_update_data( """ data = {} fields = [ - "llm_custom_setting", + "llm_custom_setting" # Hidden fields: not yet implemented, uncomment when ready # "timezone", # "boundary_detection_timeout", diff --git a/methods/EverCore/uv.lock b/methods/EverCore/uv.lock index b8d4ea65..a82c451e 100644 --- a/methods/EverCore/uv.lock +++ b/methods/EverCore/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = "==3.12.*" [[package]] @@ -1531,11 +1531,15 @@ dev = [ { name = "isort" }, { name = "nest-asyncio" }, { name = "pre-commit" }, + { name = "psutil" }, { name = "py-spy" }, { name = "pyinstrument" }, + { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "ruff" }, + { name = "ty" }, { name = "typer" }, ] dev-full = [ @@ -1544,11 +1548,15 @@ dev-full = [ { name = "isort" }, { name = "nest-asyncio" }, { name = "pre-commit" }, + { name = "psutil" }, { name = "py-spy" }, { name = "pyinstrument" }, + { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "ruff" }, + { name = "ty" }, { name = "typer" }, ] evaluation = [ @@ -1628,11 +1636,15 @@ dev = [ { name = "isort", specifier = ">=6.0.1" }, { name = "nest-asyncio" }, { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "psutil", specifier = ">=7.0.0" }, { name = "py-spy" }, { name = "pyinstrument" }, + { name = "pyright", specifier = ">=1.1.400" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.0.0" }, + { name = "ruff", specifier = ">=0.15.0" }, + { name = "ty", specifier = ">=0.0.39" }, { name = "typer" }, ] dev-full = [ @@ -1641,11 +1653,15 @@ dev-full = [ { name = "isort", specifier = ">=6.0.1" }, { name = "nest-asyncio" }, { name = "pre-commit", specifier = ">=4.3.0" }, + { name = "psutil", specifier = ">=7.0.0" }, { name = "py-spy" }, { name = "pyinstrument" }, + { name = "pyright", specifier = ">=1.1.400" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.0.0" }, + { name = "ruff", specifier = ">=0.15.0" }, + { name = "ty", specifier = ">=0.0.39" }, { name = "typer" }, ] evaluation = [ @@ -2030,6 +2046,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/cc/7e77861000a0691aeea8f4566e5d3aa716f2b1dece4a24439437e41d3d25/protobuf-5.29.5-py3-none-any.whl", hash = "sha256:6cf42630262c59b2d8de33954443d94b746c952b01434fc58a417fdbd2e84bd5", size = 172823, upload-time = "2025-05-28T23:51:58.157Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + [[package]] name = "psycopg" version = "3.3.1" @@ -2280,6 +2312,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/b4/286c12fa955ae0597cd4c763d87c986e7ade681d4b11a81766f62f079c79/pymongo-4.15.5-cp312-cp312-win_arm64.whl", hash = "sha256:649cb906882c4058f467f334fb277083998ba5672ffec6a95d6700db577fd31a", size = 896357, upload-time = "2025-12-02T18:43:08.801Z" }, ] +[[package]] +name = "pyright" +version = "1.1.409" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/4e/3aa27f74211522dba7e9cbc3e74de779c6d4b654c54e50a4840623be8014/pyright-1.1.409.tar.gz", hash = "sha256:986ee05beca9e077c165758ad123667c679e050059a2546aa02473930394bc93", size = 4430434, upload-time = "2026-04-23T11:02:03.799Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6b/330d8ebae582b30c2959a1ef4c3bc344ebde48c2ff0c3f113c4710735e11/pyright-1.1.409-py3-none-any.whl", hash = "sha256:aa3ea228cab90c845c7a60d28db7a844c04315356392aa09fafcee98c8c22fb3", size = 6438161, upload-time = "2026-04-23T11:02:01.309Z" }, +] + [[package]] name = "pytest" version = "9.0.1" @@ -2551,6 +2596,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, ] +[[package]] +name = "ruff" +version = "0.15.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" }, + { url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" }, + { url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" }, + { url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" }, + { url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" }, + { url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" }, + { url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" }, + { url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" }, +] + [[package]] name = "scikit-learn" version = "1.7.2" @@ -2796,6 +2866,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, ] +[[package]] +name = "ty" +version = "0.0.39" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/8d/7b5c74dc287fbcb37bae9853cec13bf44717c1735298500e4aeba31579a9/ty-0.0.39.tar.gz", hash = "sha256:f750277e76a01ecd86185960eca73823c26a53c51103568d56d4d904575159fd", size = 5702365, upload-time = "2026-05-22T21:09:56.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/17/9b89802c26d12d0f7a27bc25d4066d941d42891e8898f9f26499f0067e32/ty-0.0.39-py3-none-linux_armv6l.whl", hash = "sha256:c1bb7ac70f1f7d70cc6655fd96558039e4562b10f489fa49c7ebfd5fcee73ad1", size = 11360431, upload-time = "2026-05-22T21:09:18.689Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c6/663ded50e823dbf9fb9d002eca46b7cb1fb2c72b744b84f22ce732a0ee0b/ty-0.0.39-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3435b64c1e59c14c9aa39c20cc018823937cd38d55db853e74d95b8f420569b0", size = 11096281, upload-time = "2026-05-22T21:09:15.383Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ae/5d38ba9a6456ff4c78d212cf464fd8b9a25d8118465197b0b2dc891c0b19/ty-0.0.39-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5f136377ce46c73677701a9e1ad730bf72f699bcec046e422eb79d0886cac3ab", size = 10529674, upload-time = "2026-05-22T21:09:46.471Z" }, + { url = "https://files.pythonhosted.org/packages/be/6f/43638cb8106445d3c8817256a0731cde9dd7b6a53ae2e881294bc1930ca3/ty-0.0.39-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36b65fb0cc17f03e851d40e210d420be94ab8bc52d041328ad1e45f616036a61", size = 11055561, upload-time = "2026-05-22T21:09:36.981Z" }, + { url = "https://files.pythonhosted.org/packages/91/17/95e62cf4458527ce78dc386eba18f8b10c3fb64cd8c9e7e59b262ff6029d/ty-0.0.39-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4967967bfadf3860ff84c3fccdbaec8edf8aa20d0d727521084733d853de6657", size = 11127185, upload-time = "2026-05-22T21:09:31.395Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c0/93666c213db5c71ab1b1f1a0db5f66bf8c7c0e0b0bf59859f5da8f0b3c36/ty-0.0.39-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e10ecb1297099ddf9a1f054f8bd921d1863ce85fb819a3c96ed27865a1ba6ed", size = 11608459, upload-time = "2026-05-22T21:09:12.862Z" }, + { url = "https://files.pythonhosted.org/packages/79/85/3b26585afc8b50230d6464bb0642feef4fab3f847e38b1f0ffa971a81446/ty-0.0.39-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9b19cca70e465d71b0510656343883d62372bbe74b7845cae7c0e701d6d5264b", size = 12177101, upload-time = "2026-05-22T21:09:40.519Z" }, + { url = "https://files.pythonhosted.org/packages/49/4a/1039e4f6afc576dc1c3a4d22a6478904a1ad3766597cd0b93c077ab9dfce/ty-0.0.39-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:56c6704b01b9b3d80ff26b2918423b742516d1e469bef830e9254dcedc9185bf", size = 11827815, upload-time = "2026-05-22T21:09:49.89Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c5/4688652870e350a76a8157f7ffb59ad54f37d5d10725aa7076f66ac94ec8/ty-0.0.39-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40b7840ff46764b6a6757f4ade1cd0530fc3e8a0b435ca93e7602360e4cb90b6", size = 11694429, upload-time = "2026-05-22T21:09:21.568Z" }, + { url = "https://files.pythonhosted.org/packages/fc/72/8a1c4e823bb5bdc935a1c8140e100304e36a68a4139592f170aa9736fdb7/ty-0.0.39-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:1c62a3a87ce26b50819f0dbf03bd95f23f19eeb87bbc7aa732ec64277c77f1aa", size = 11869846, upload-time = "2026-05-22T21:09:28.053Z" }, + { url = "https://files.pythonhosted.org/packages/17/9f/cf982457b861ae22d657c5dcdbc631199f7f90264279db1d17230dfbc3ff/ty-0.0.39-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f8c34bc81a9c3516e49904e9d8330aac385377cca98390193ea02b903a40fcf0", size = 11029763, upload-time = "2026-05-22T21:09:06.791Z" }, + { url = "https://files.pythonhosted.org/packages/46/c9/95b64f6d43ae6e8f0b7e13dacf9c196d35819af22b1924171fba31383156/ty-0.0.39-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:66f5ab11586a64e79cb692ad685ee5469325c31b5f30bd3554f52f36dbe28cc4", size = 11146761, upload-time = "2026-05-22T21:09:10.178Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/0a89cfb06f7632a05bf56c78e0affb4a40f81759e275376cea75c9c5abe9/ty-0.0.39-py3-none-musllinux_1_2_i686.whl", hash = "sha256:e8d89732bcbbcb091f439e556dfc4932f198b118b47d5b85212c60662099670e", size = 11281843, upload-time = "2026-05-22T21:09:34.234Z" }, + { url = "https://files.pythonhosted.org/packages/0e/53/64c4a27067a46643fea2b3fcf21a8a2f838d91a65ffdd14f2e82945b9538/ty-0.0.39-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:eceb6c91dcd05a231119f82abdd9aa337513de23ca6ac990bc44f88791dc1799", size = 11792477, upload-time = "2026-05-22T21:09:24.923Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e8/02f4dd4a12bcdbda0006f9c7ff3b99a4be06bd0d257d3bd4a5b66de074e6/ty-0.0.39-py3-none-win32.whl", hash = "sha256:891c3262314dbc80bf3e872634d23dd216306945daa9a9fcc206ce5ed21ac4c9", size = 10615377, upload-time = "2026-05-22T21:09:43.167Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5a/aaeb22faa8d4dae90a287d4c3636c671edcff3b99be5f4fc8b79ad71eef6/ty-0.0.39-py3-none-win_amd64.whl", hash = "sha256:ba7f2d54452535419e90f6f03ff39282999e87b43c21c00559f6d7ad711a36d5", size = 11710711, upload-time = "2026-05-22T21:09:53.179Z" }, + { url = "https://files.pythonhosted.org/packages/a3/17/ae7339651bfcaa5f54698c8c70eaf5031baa400ecb67baec31d03a56cbd4/ty-0.0.39-py3-none-win_arm64.whl", hash = "sha256:eb4cf0fefbbfedf9a352597bb2431ebdcb7eb3a595c0f825f228e897a0ec285d", size = 11081409, upload-time = "2026-05-22T21:09:03.741Z" }, +] + [[package]] name = "typer" version = "0.20.0" From 2380d81d708bbeda605cbe279a419099cfd2a164 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:06:11 +0800 Subject: [PATCH 11/24] Gitignore CLAUDE.local.md for per-developer overrides Claude Code loads CLAUDE.local.md alongside CLAUDE.md and lets it take precedence (closer-to-cwd loads last). Use it for personal workflow rules that shouldn't bind other contributors, while CLAUDE.md / AGENTS.md keep the shared team conventions. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 72156a90..9038eacd 100755 --- a/.gitignore +++ b/.gitignore @@ -209,6 +209,9 @@ AGENTS.mk docs/api_docs/profile_extraction_fields.md .claude/ .cursor/* +# Per-developer CLAUDE.md overrides (loaded by Claude Code alongside CLAUDE.md) +CLAUDE.local.md +**/CLAUDE.local.md #tmp_data demo/memcell_outputs/ From 9cf4b380160ffebc38366d9363c49d904535963f Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:06:14 +0800 Subject: [PATCH 12/24] docs(EverCore): add exception handling audit and code quality roadmap Two new dev_docs articles drafted from a project-wide code review: - exception_handling_analysis.md: full audit of 781 try blocks and 17 hand-written retry loops. Catalogues 7 anti-patterns and 6 retry types with code references, performance/correctness analysis, and a sprint-organised action plan. - code_quality_roadmap.md: 8-week three-phase execution plan (observability -> tests -> try-catch cleanup), with per-phase acceptance criteria and rollback paths. --- .../docs/dev_docs/code_quality_roadmap.md | 394 +++++++++ .../dev_docs/exception_handling_analysis.md | 762 ++++++++++++++++++ 2 files changed, 1156 insertions(+) create mode 100644 methods/EverCore/docs/dev_docs/code_quality_roadmap.md create mode 100644 methods/EverCore/docs/dev_docs/exception_handling_analysis.md diff --git a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md new file mode 100644 index 00000000..9d26f42e --- /dev/null +++ b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md @@ -0,0 +1,394 @@ +# Code-Quality Improvement Roadmap + +**Project**: EverCore +**Drafted**: 2026-05-26 +**Estimated duration**: 6-8 weeks +**Companion document**: [`exception_handling_analysis.md`](./exception_handling_analysis.md) + +--- + +## 1. Background and Strategy + +### 1.1 Why now + +The code audit (see [`exception_handling_analysis.md`](./exception_handling_analysis.md)) surfaced systemic issues: + +- **Exception handling**: 781 try blocks, 71% are `except Exception` catch-alls, the largest try block is 241 lines +- **Retry strategy**: 6 copy-pasted HTTP retries, 5 LLM retries with **no sleep at all**, 2 parallel `RetryConfig` classes +- **Tests**: 68 test files and 47 K lines of test code — yet **CI does not run `pytest` at all** +- **Observability**: 94 Prometheus metrics exist, but **structured logging and distributed tracing are entirely missing**, and `/health` is only a shallow check + +### 1.2 Three-Phase Strategy + +``` +┌────────────────────────────────────────────────────────────────────┐ +│ Phase 1: Observability (2 weeks) → Instrument before driving │ +│ ↓ │ +│ Phase 2: Test foundation (2 weeks) → Build the safety net │ +│ ↓ │ +│ Phase 3: try-catch cleanup (3 weeks) → Cut, measure the effect │ +└────────────────────────────────────────────────────────────────────┘ +``` + +**Guiding principle: observe first, measure next, change last.** + +**Why not in another order?** + +- ❌ Refactor try-catch first with no metrics → no way to know whether things got better, error-rate movement is unattributable +- ❌ Refactor first with no tests → bugs surface only after deployment +- ❌ Add tests first with no metrics → tests don't know what "stable" should look like (no SLO baseline) + +Each phase prepares the ground for the next. Each phase **delivers value on its own** — even stopping at Phase 1 leaves the system better than today. + +### 1.3 Success Criteria (end state) + +| Dimension | Current | Target | +|---|---|---| +| `except Exception` share | 71% | < 30% | +| Largest try block | 241 lines | < 50 lines | +| CI runs unit tests | ❌ | ✅ | +| Code coverage measured | Not measured | Report + 60% threshold | +| Structured logging | ❌ (plain strings) | ✅ JSON output | +| Distributed tracing | ❌ | ✅ OpenTelemetry | +| `/readyz` vs `/livez` | Not separated | Separated | +| Downstream health checks | ❌ | All 5 dependencies probed | + +--- + +## 2. Phase 1: Observability (2 weeks) + +**Goal**: Before changing any code, make the system's internal state **visible** and **measurable**. + +### 2.1 Current State + +✅ Already in place: +- 94 Prometheus metrics (memorize, retrieve, vectorize, rerank) +- `@trace_logger` decorator + `stage_timer` +- Request ID propagation through middleware +- 23 `contextvars` usages + +❌ Missing: +- Structured logging (all 247 logger calls use plain f-strings) +- OpenTelemetry / distributed tracing +- Health check that doesn't validate downstreams +- SLO/SLI definitions +- Error metric labelled by `exception_type` + +### 2.2 Deliverables + +#### P0: Must-do (week 1) + +**T1.1 Split health check into `/livez` and `/readyz`** +- File: `src/infra_layer/adapters/input/api/health/health_controller.py` +- Changes: + - `/livez`: process heartbeat; no downstream checks; always 200 + - `/readyz`: verify Redis + MongoDB + Elasticsearch + Milvus + LLM provider can all be reached + - Emit a separate metric `dependency_healthy{name="..."}` per downstream +- Acceptance: K8s probe configuration documented +- Effort: half a day + +**T1.2 Structured logging (structlog)** +- Add `structlog>=24.1.0` to dependencies +- Replace the `get_logger` implementation in `core/observation/logger.py` +- Output format: coloured console in development, JSON in production +- Standard fields: `request_id`, `tenant_id`, `user_id`, `group_id`, `session_id` +- **Don't touch logger call sites yet** — keep backward compatibility; the migration is for Phase 3 +- Effort: 1-2 days + +**T1.3 Error metrics labelled by exception type** +- Update the existing `record_*_error` functions: change the `error_type` label from `'unknown'` to the concrete exception class name +- Add a `logger.exception` helper in `core/observation/logger.py` that automatically emits the metric +- Effort: half a day + +#### P1: Strongly recommended (week 2) + +**T1.4 OpenTelemetry integration** +- Dependencies: + ``` + opentelemetry-api>=1.27.0 + opentelemetry-sdk>=1.27.0 + opentelemetry-instrumentation-fastapi + opentelemetry-instrumentation-httpx + opentelemetry-instrumentation-redis + opentelemetry-instrumentation-pymongo + opentelemetry-exporter-otlp + ``` +- Modify the `@trace_logger` decorator: emit real OTel spans, not just logs +- Configuration: `OTEL_EXPORTER_OTLP_ENDPOINT` env var, disabled by default +- Add Jaeger / Tempo to docker-compose +- Effort: 2-3 days + +**T1.5 SLO/SLI definitions document** +- Create `docs/dev_docs/slo_definitions.md` +- Define: + - `memorize_success_rate` (5xx rate < 0.1%) + - `retrieve_p95_latency` (< 500 ms) + - `llm_call_error_budget` (< 1% failures per month) +- Commit Grafana dashboard JSON to `docs/grafana/` +- Effort: 1 day + +### 2.3 Phase 1 Acceptance + +| Check | How to verify | +|---|---| +| `/livez` returns 200 with no downstream dependencies | `curl localhost:1995/livez` | +| `/readyz` returns 503 after Redis goes down | `docker stop memsys-redis && curl /readyz` | +| Default log output is JSON | `tail logs/*.log \| jq .` doesn't fail | +| Error metric `exception_type` label is non-empty | `curl /metrics \| grep memorize_error` | +| Jaeger UI shows a complete trace | Open `:16686` and trigger a memorize request | + +### 2.4 Exit Criteria + +✅ All P0 items plus at least one P1 item complete +✅ No production incidents +✅ The team can read Grafana / Jaeger (training done) + +--- + +## 3. Phase 2: Test Foundation (2 weeks) + +**Goal**: Build the "change-without-fear" safety net, especially in preparation for the semantic refactoring in Phase 3. + +### 3.1 Current State + +✅ Already in place: +- 68 test files, 47,061 lines of test code +- `pytest-asyncio` and `pytest-cov` are dev deps +- 785 fixture / mock usages +- 32 E2E tests (depend on docker-compose) + +❌ Missing: +- **CI does not run tests** (biggest problem) +- No coverage report +- 32 of 68 tests are heavy E2E; unit-test share is too low +- Error-path tests are sparse (71% catch-all makes error paths untestable) +- No performance benchmarks + +### 3.2 Deliverables + +#### P0: Must-do (week 3) + +**T2.1 CI runs pytest** +- Modify `.github/workflows/evercore-smoke.yml`: add a `test` job +- Start docker-compose services (Redis, MongoDB, ES, Milvus) +- Run `make test` +- Effort: 1 day (including CI debugging) + +**T2.2 Enable coverage reporting with a threshold** +- Change `make test` to `pytest --cov=src --cov-report=xml --cov-report=term` +- Wire codecov or GitHub PR coverage comments into CI +- Treat the current baseline as the minimum (do not allow regression) +- Effort: half a day + +**T2.3 Test classification markers** +- Tag the 68 tests with `@pytest.mark.unit` / `@pytest.mark.integration` / `@pytest.mark.e2e` +- CI runs `unit + integration` by default; E2E runs in its own job or manually +- Local `make test-unit` runs without docker +- Effort: 1-2 days + +#### P1: Strongly recommended (week 4) + +**T2.4 Backfill unit tests on critical paths** +- Priority modules (aligned with Phase 3 refactor priorities): + - `biz_layer/mem_memorize.py` (home of the 241-line try) + - `agentic_layer/memory_manager.py` (source of the possibly-unbound bug) + - `memory_layer/memory_extractor/agent_skill_extractor.py` (Type B retry anti-pattern) +- Target: happy-path + at least 3 error paths per module +- Mock strategy: mock `LLMProvider` and repositories; **do not mock business logic** +- Effort: 3-5 days + +**T2.5 Type checking in CI** +- Add `make typecheck` to CI (runs `ty check`) +- Treat the current 891 ty diagnostics as baseline; do not allow increases +- Effort: 30 minutes + +**T2.6 Performance benchmark baseline** +- Add a `tests/benchmarks/` directory +- Use `pytest-benchmark` +- Key paths: + - `retrieve_mem_hybrid` p50/p95 over 100 calls + - End-to-end `memorize` latency for a single message +- Don't block CI yet (establish baseline first; add thresholds in 3 months) +- Effort: 1-2 days + +### 3.3 Phase 2 Acceptance + +| Check | How to verify | +|---|---| +| The `pytest` step in CI shows green | GitHub Actions page shows ✅ | +| Coverage report posted on each PR automatically | Open a test PR | +| `make test-unit` finishes locally in under 30 s | Time it | +| `make typecheck` does not report new errors | Run in CI | +| Line coverage of `biz_layer/mem_memorize.py` > 70% | Coverage report | + +### 3.4 Exit Criteria + +✅ All P0 items plus T2.4 (unit-test backfill) complete +✅ Coverage baseline established +✅ At least 3 PRs have been caught and fixed because of test failures (proving CI works) + +--- + +## 4. Phase 3: try-catch Cleanup (3 weeks) + +**Goal**: Put the tooling built in Phases 1 and 2 to work and **change real business code**. This is the highest-volume, highest-risk phase — but with the preparation done, the change is measurable and reversible. + +### 4.1 Entry Conditions + +✅ Phase 1 complete: error metrics labelled by type, tracing available +✅ Phase 2 complete: CI runs tests, coverage threshold enforced +✅ Critical modules have unit tests + +### 4.2 Deliverables + +See [`exception_handling_analysis.md` §9 Action Plan](./exception_handling_analysis.md). Execution cadence: + +#### Week 5: Low-risk mechanical changes + +Per the audit report, Sprint 1: +- Replace the 20 `traceback.print_exc()` with `logger.exception` / `exc_info=True` +- Enable `ruff G004` and clean up 247 f-string logger calls +- Merge the two `RetryConfig` classes +- Enable `ruff BLE001` baseline + +Each item is its own PR, and **each PR compares Phase 1 metrics before/after** (error rate, counts per `exception_type`). + +#### Week 6: Retry refactor + +- Type A: 6 HTTP retries → tenacity decorator +- Type B: 5 LLM output-format retries → "feedback-retry" +- Type E: service-level fallback → add a lock or adopt `aiobreaker` + +Each as its own PR, and **the retry-count metrics in Grafana should stay flat**. + +#### Week 7: Business-layer catch-all overhaul + +Per the audit report, Sprints 2-3: +- Introduce a custom exception hierarchy (`MemorizeError`, `RetrieveError`, `ExtractionError`) +- Convert business-layer catch-all to upward propagation +- Add boundary except blocks in controllers to catch the specific types +- Split the top-5 oversize try blocks + +This is the riskiest week. **Every PR must**: +1. Pass full CI with all tests green +2. Not lower coverage +3. Run 24 hours in staging with error rate not rising + +### 4.3 Phase 3 Acceptance + +| Check | Current | Target | +|---|---|---| +| `except Exception` share | 71% | < 30% | +| `traceback.print_exc()` remaining | 20 | 0 | +| Largest try block (lines) | 241 | < 50 | +| logger f-string count | 247 | < 50 (keep legitimate uses like prompt templates) | +| `error_type='unknown'` share | ~80% | < 10% | + +### 4.4 Exit Criteria + +✅ All P0/P1 items complete +✅ Production error rate has not risen (30-day comparison) +✅ MTTR (mean time to repair) is at least flat or has fallen + +--- + +## 5. Timeline Overview + +``` +Week 1-2 │ Phase 1: Observability + │ W1: /livez+readyz, structlog, error metrics + │ W2: OpenTelemetry, SLO document + │ +Week 3-4 │ Phase 2: Test foundation + │ W3: CI pytest, coverage, test markers + │ W4: Unit-test backfill, typecheck in CI, benchmark baseline + │ +Week 5-7 │ Phase 3: try-catch cleanup + │ W5: Mechanical fixes (traceback / f-string / RetryConfig) + │ W6: Retry refactor (tenacity / circuit breaker) + │ W7: Catch-all + business exceptions + large try-block splits + │ +Week 8 │ Buffer / wrap-up / documentation +``` + +**Total effort estimate**: 8 weeks (one engineer full-time) or 12 weeks (half-time). + +--- + +## 6. Risk and Rollback + +### 6.1 Top Risks + +| Risk | Probability | Impact | Mitigation | +|---|---|---|---| +| Phase 1 structlog rollout breaks log format | Low | Medium | Canary release, keep the old logger as a fallback for 1 week | +| Phase 2 docker-based CI is slow | Medium | Low | Run unit and integration tests in parallel jobs | +| Phase 3 surfaces the real error rate | High | Medium | **That's a feature**; the rate was always there, just hidden. Notify oncall first. | +| Phase 3 retry changes affect LLM bill | Medium | Low | Add a retry-budget cap metric and alert on it | +| OpenTelemetry adds overhead | Low | Low | Default sampling rate 10%, configurable | + +### 6.2 Per-Phase Rollback + +- **Phase 1**: every change is additive (structlog, tracing, new endpoints). Each can be turned off behind a feature flag. +- **Phase 2**: CI changes roll back in seconds (`git revert` on the workflow yaml). +- **Phase 3**: every PR is independently revertable; critical changes stay behind feature flags. + +--- + +## 7. How We'll Measure Improvement + +The "success picture" for the 8-week effort: + +``` +Error visibility: + error_type='unknown' share 80% → < 10% + Sentry/alert grouping coarse → fine-grained (by exception class) + +Debug efficiency: + MTTR median flat or downward + Time to locate a root cause minutes → seconds in Grafana + +Code health: + ruff baseline 239 → < 50 + ty baseline 891 → < 800 (just no growth) + Coverage unknown → > 60% + except Exception share 71% → < 30% + +Engineering efficiency: + Local unit-test feedback loop none → < 30 seconds + CI-to-deploy manual → automatic + "Dare I change one line?" low → high +``` + +--- + +## 8. Explicitly Out of Scope + +To keep this plan focused, the following are **not** part of this effort: + +- ❌ Switching Python versions (stay on 3.12) +- ❌ Switching frameworks (stay on FastAPI / Pydantic v2) +- ❌ Data-model refactor (Pydantic schemas stay put) +- ❌ Redesigning the RAG algorithm +- ❌ Multi-region deployment / DR design +- ❌ Performance optimisation (unless Phase 3 happens to expose a clear bottleneck) + +These are all worthwhile — they're just not part of the "code quality" main line. + +--- + +## 9. Next Steps + +1. **This week**: the team reviews the roadmap and locks owners / priorities +2. **Next Monday**: Phase 1 W1 kicks off; first PR is T1.1 `/livez` + `/readyz` +3. **Every Friday**: progress sync, risk review, decide whether ordering needs to shift + +If you have questions or want to adjust ordering or drop items, leave a PR comment below or open a GitHub issue. + +--- + +## Appendix: Related Documents + +- [`exception_handling_analysis.md`](./exception_handling_analysis.md) — Full exception-handling and retry audit (data foundation) +- [`development_standards.md`](./development_standards.md) — Project development standards +- [`metrics_library_design.md`](./metrics_library_design.md) — Existing metrics library design diff --git a/methods/EverCore/docs/dev_docs/exception_handling_analysis.md b/methods/EverCore/docs/dev_docs/exception_handling_analysis.md new file mode 100644 index 00000000..b1975037 --- /dev/null +++ b/methods/EverCore/docs/dev_docs/exception_handling_analysis.md @@ -0,0 +1,762 @@ +# Exception Handling Audit Report + +**Project**: EverCore (`methods/EverCore/`) +**Audit date**: 2026-05-26 +**Audit scope**: All of `src/**/*.py` +**Tools used**: `grep` / `ruff` / `ty` / Python AST + +--- + +## 1. TL;DR + +| Metric | Value | Assessment | +|---|---|---| +| Total `try` blocks | **781** | High | +| Total functions | **2,448** | — | +| **`try` blocks / function ratio** | **32%** | Industry RAG/Agent projects typically sit at 10-20% | +| `except Exception` share | **555 / 781 = 71%** | 🔴 Severely elevated; healthy range is < 30% | +| `logger` calls using f-strings | **247** | 🟡 CPU waste + inconsistent with structured logging | +| `traceback.print_exc()` left in code | **20** | 🟡 Bypasses the logger pipeline | +| Largest single `try` block | **241 lines** (`mem_memorize.py:118`) | 🔴 Practically wraps the entire function | +| Average `try` block size | 18.9 lines | Too large (ideal < 5 lines) | +| Hand-written retry loops (`for attempt in range`) | **17** | 🟡 None use `tenacity` — wheels reinvented | +| `2 ** attempt` backoff with no jitter | **~10** | 🔴 Thundering-herd risk | +| Duplicated `RetryConfig` classes | **2** | 🟡 One in `core/longjob`, one in `core/asynctasks` | + +**Bottom line**: The codebase exhibits widespread "defensive-programming overreach". From CPython 3.11 onward `try` blocks themselves carry **zero overhead** on the happy path, so this **is not a performance problem** — it is a **correctness and observability problem**: errors are silently swallowed at scale, bugs struggle to surface, and the system runs in a degraded state without callers being aware. Retry handling is in equally bad shape: six near-identical copies of exponential backoff, five LLM retries with **no sleep whatsoever**, and two parallel `RetryConfig` classes coexisting. + +--- + +## 2. Data Portrait + +### 2.1 Density by Architectural Layer + +| Layer | `try` blocks | Functions | Density | Assessment | +|---|---|---|---|---| +| `biz_layer` | 33 | 46 | **72%** | 🔴 The business layer should not be this high | +| `infra_layer` | 190 | 262 | **73%** | 🟡 Boundary code (DB/ES/Milvus) running high is somewhat expected, but the except shapes need work | +| `memory_layer` | 54 | 200 | 27% | 🟢 Acceptable | +| `agentic_layer` | 50 | 202 | 25% | 🟢 Acceptable | +| `core` | 359 | 1,426 | 25% | 🟢 Acceptable | +| `service` | 8 | 43 | 19% | 🟢 Good | +| `common_utils` | 15 | 85 | 18% | 🟢 Good | +| `api_specs` | 9 | 65 | 14% | 🟢 Good (a DTO layer should be light) | + +**Key observation**: `biz_layer` is the standout anomaly. The business-logic layer is supposed to let exceptions propagate naturally to its callers (controllers), not absorb them itself. + +### 2.2 Top 10 Files by `try` Density + +| File | `try` blocks | +|---|---| +| `infra_layer/.../memcell_raw_repository.py` | 24 | +| `core/queue/.../redis_msg_group_queue_manager.py` | 23 | +| `biz_layer/mem_memorize.py` | 20 | +| `agentic_layer/retrieval_utils.py` | 14 | +| `infra_layer/.../raw_message_repository.py` | 13 | +| `devops_scripts/i18n/i18n_tool.py` | 13 | +| `core/oxm/es/base_repository.py` | 13 | +| `core/component/redis_provider.py` | 13 | +| `infra_layer/.../agent_case_raw_repository.py` | 12 | +| `core/oxm/milvus/milvus_collection_base.py` | 12 | + +### 2.3 Top 5 Largest `try` Blocks (the "over-wrapping" anti-pattern) + +| File:line | Size | Nature | +|---|---|---| +| `biz_layer/mem_memorize.py:118` (`_trigger_clustering`) | **241 lines** | The whole function body is wrapped | +| `core/queue/msg_group_queue/msg_group_queue_manager.py:553` | 182 lines | | +| `core/lock/redis_distributed_lock.py:67` | 175 lines | | +| `agentic_layer/memory_manager.py:617` (`get_vector_search_results`) | 153 lines | Essentially the entire function body | +| `memory_layer/memory_extractor/agent_case_extractor.py:594` | 148 lines | | + +--- + +## 3. Anti-Pattern Catalogue (with code references) + +### Anti-Pattern 1: Catch-all swallows real bugs into a black hole + +**Symptom**: A large block of logic is wrapped in `except Exception`; the only response is `logger.error` followed by returning a "safe default". + +**Example 1**: `biz_layer/mem_memorize.py:1857-1860` +```python +try: + for memcell in memcells: + await process_memory_extraction(memcell, ...) + return memories_count +except Exception as e: + logger.error(f"[mem_memorize] ❌ Memory extraction failed: {e}") + traceback.print_exc() + return 0 # ← controller treats 0 as "no new memories"; actually it's a failure +``` + +The API side at `memory_controller.py:165` consumes this return value to decide status: +```python +status = 'extracted' if memory_count > 0 else 'accumulated' +``` + +**Effect**: A write failure becomes HTTP 200 + `"status": "accumulated"` — **data is lost with no alarm signal**. + +**Historical cost**: The `milvus_start` `NameError` bug found in an earlier audit (`memory_manager.py:782`) was swallowed by exactly this pattern. Code that should have crashed only emitted a single `error` log line in production, while the original exception was fully concealed. + +--- + +### Anti-Pattern 2: Nested-recovery try — Russian-doll handlers + +**Symptom**: An `except` block contains its own "recovery operation" wrapped in another try; if the recovery also fails, the failure is swallowed again. + +**Example**: `biz_layer/mem_memorize.py:528-556` `_trigger_profile_extraction` +```python +try: + # ... main profile-extraction logic ... +except Exception as e: + logger.error(f"[Profile] ❌ Profile extraction failed: {e}", exc_info=True) + try: + # ← nested try: advance last_updated_ts on failure to avoid infinite re-selection + for uid in user_id_list: + await profile_repo.upsert(...) + except Exception as ts_err: + logger.warning(f"[Profile] Failed to advance last_updated_ts: {ts_err}") + # ← the second-level failure is silently swallowed +``` + +**Problem**: The control flow reads like recursion. Once the second `except` fires, no signal escapes; the problem is permanently invisible. + +--- + +### Anti-Pattern 3: Exceptions downgraded to "fake success" + +**Symptom**: The function returns `None` / `[]` / `0` / `False` / the original input on error, so callers cannot distinguish "no data" from "fetch failed". + +**Project-wide scan**: high concentration under `memory_extractor/`: + +| File:line | Returns on failure | +|---|---| +| `foresight_extractor.py:320` | `[]` | +| `foresight_extractor.py:358` | `None` | +| `foresight_extractor.py:383` | `None` | +| `agent_skill_extractor.py:295` | `None` | +| `agent_skill_extractor.py:336` | `None` | +| `agent_skill_extractor.py:382` | `None` | +| `agent_skill_extractor.py:788` | `[]` | +| `episode_memory_extractor.py:403` | `None` | + +**Specific case**: `biz_layer/mem_memorize.py:1432-1436` `preprocess_conv_request`: +```python +except Exception as e: + logger.error(f"[preprocess] Data read failed: {e}") + traceback.print_exc() + # Use original request if read fails + return request # ← returns the unprocessed request as if it had been processed +``` + +Downstream code receiving `request.history_raw_data_list` may see an empty list or stale data with no metric or exception telling it so. + +--- + +### Anti-Pattern 4: Over-wrapped try + +**Canonical example**: `biz_layer/mem_memorize.py:118` `_trigger_clustering` — a **241-line** try that wraps the entire function body. + +Meaning: any line inside the function that throws enters the same `except`, and there is no way to distinguish "DI lookup failed" from "clustering computation failed" from "database write failed". When you go to fix something, you can only guess. + +**Ideal granularity**: a try should hug **one** specific external call that can fail (DB / API / IO). For example: + +```python +# Don't do this +try: + config = load_config() # ← can't fail + data = fetch_from_db() # ← can fail + result = process(data) # ← pure compute, shouldn't be in try + save_to_db(result) # ← can fail +except Exception: + pass + +# Do this +config = load_config() +try: + data = fetch_from_db() +except DBError as e: + raise FetchFailure(...) from e + +result = process(data) + +try: + save_to_db(result) +except DBError as e: + raise PersistenceFailure(...) from e +``` + +--- + +### Anti-Pattern 5: Defending against exceptions that cannot happen + +**Symptom**: a try wraps pure-Python operations (dict lookups, string strip, list indexing) that don't raise. + +**Example**: `agentic_layer/retrieval_utils.py:233-241` (already changed to `except Exception:`, but the try itself remains) +```python +for mem in candidates: + try: + doc_vec = np.array(mem.extend.get("embedding", [])) + if len(doc_vec) > 0: + doc_norm = np.linalg.norm(doc_vec) + if doc_norm > 0: + sim = np.dot(query_vec, doc_vec) / (query_norm * doc_norm) + scores.append((mem, float(sim))) + except Exception: + continue # ← wrapping a numpy dot product in try-except is over-defensive +``` + +`np.dot` does not raise on shape-matching float arrays. This try is really shielding **type errors**, but type errors should be allowed to crash and be caught by the type checker (ty/pyright), not silently skipped at runtime. + +--- + +### Anti-Pattern 6: `traceback.print_exc()` bypassing structured logging + +**20 occurrences remain**. This function: +- Writes directly to stderr +- Does not flow through the logger pipeline, so it never reaches structured logs / alerting systems +- May be lost in containerised environments where stderr/stdout is redirected + +**Right way**: +```python +# Anti-pattern +except Exception as e: + logger.error(f"failed: {e}") + traceback.print_exc() + +# Correct +except Exception as e: + logger.error("failed: %s", e, exc_info=True) +``` + +`exc_info=True` hands the full stack to the logger formatter, which uniformly delivers it into structured logs. + +--- + +### Anti-Pattern 7: f-strings inside `logger` calls + +**247 occurrences project-wide.** +```python +# Anti-pattern +logger.debug(f"query_words: {query_words}") # the f-string is evaluated even if DEBUG is off + +# Correct +logger.debug("query_words: %s", query_words) # interpolation only when DEBUG is enabled +``` + +**Performance angle**: every call interpolates the string (formatting `repr()` of lists/dicts is real CPU work). On hot paths it accumulates, but **the per-call cost is small**. + +**Consistency angle**: `run.py` uses `%`-style while the other 99% of modules use f-strings. This is a team-convention issue. + +--- + +## 4. Retry-Strategy Audit + +If the previous anti-patterns are "one-shot try gone wrong", retries are "repeated try gone wrong" — more systemic and more hidden. + +### 4.1 Overall Picture + +| Type | Nature | Occurrences | Main problem | +|---|---|---|---| +| **A. HTTP/API call retry** | Infrastructure layer | ~8 | Copy-pasted `2**attempt`, no jitter, no exception-type distinction | +| **B. LLM output-format retry** | Business layer | 5 | **No sleep at all**, immediate retry — burns money + stacks with A | +| **C. Distributed-lock acquire retry** | Lock contention | 1 | Reasonable; no issue | +| **D. Background-task retry** (ARQ) | Worker framework | 2 separate `RetryConfig` classes | Wheel reinvented | +| **E. Service-level fallback** (pseudo-retry) | Failover | 2 | Lock-free counters; race conditions | +| **F. Loop-local try/continue** ("skip-on-failure") | Batch processing | ~6 | Partial failures are invisible | + +**Every retry is hand-written; zero `tenacity` usage**: + +```bash +grep -rn "from tenacity\|@retry\b" src/ --include="*.py" | wc -l # → 0 +``` + +### 4.2 Type A: HTTP/API Call Retry (copy-paste) + +**Template**: `for attempt in range(max_retries): try ... except: await asyncio.sleep(2**attempt)` + +| File:line | Retries | Backoff | Jitter | Exception-type discrimination | +|---|---|---|---|---| +| `core/component/llm/llm_adapter/anthropic_adapter.py:88` | `self.max_retries` | `2**attempt` | ❌ | ✅ (5xx retried, 4xx re-raised) | +| `core/component/llm/llm_adapter/gemini_adapter.py:63` | `self.max_retries` | `2**attempt` | ❌ | ❌ | +| `core/component/llm/llm_adapter/gemini_client.py:109` | `self.max_retries` | `2**attempt` | ❌ | ❌ | +| `agentic_layer/vectorize_base.py:108` | `config.max_retries` | `2**attempt` | ❌ | ❌ | +| `agentic_layer/rerank_deepinfra.py:104` | `config.max_retries` | `2**attempt` | ❌ | ❌ | +| `agentic_layer/rerank_vllm.py:...` | `config.max_retries` | `2**attempt` | ❌ | ❌ | + +**Representative code** (`gemini_adapter.py:63-85`): +```python +for attempt in range(self.max_retries): + try: + response = await self.client.aio.models.generate_content(...) + return self._convert_gemini_response(response, request.model) + except Exception as e: + if attempt == self.max_retries - 1: + raise RuntimeError( + f"An unexpected error occurred in GeminiAdapter: {e}" + ) from e + await asyncio.sleep(2**attempt) +``` + +**Problem analysis**: +1. **Copy-paste**: 6 places, nearly character-for-character identical. Maintenance cost ×6; any bug fix must be applied 6 times. +2. **No jitter**: when N concurrent requests are rate-limited at once they will retry in lock-step → a thundering herd that re-hammers the backend. +3. **No exception-type discrimination** (except Anthropic): 401 (auth) / 400 (malformed request) also gets retried N times, wasting quota. +4. **Loses the cause chain on failure**: `raise RuntimeError(f"...{e}") from e` does use `from`, but the original exception class (e.g. `APITimeoutError` vs. `RateLimitError`) is buried inside a string. +5. **2**0 = 1 s, 2**4 = 16 s**: with `max_retries=5`, the final wait is 32 s — that coroutine is blocked on the event loop for a long time. + +### 4.3 Type B: LLM Output-Format Retry (business layer, no sleep) + +**The worst-offender pattern**. Representative: `agent_skill_extractor.py:325` + +```python +for attempt in range(3): + try: + resp = await self.llm_provider.generate(prompt) + data = parse_json_response(resp) + if data and isinstance(data.get("operations"), list): + return data + logger.warning(f"... retry {attempt + 1}/3: invalid format") + except Exception as e: + logger.warning(f"... retry {attempt + 1}/3: {e}") +return None # ← all 3 attempts failed → silently return None +``` + +| File:line | Retries | Sleep | Return on failure | +|---|---|---|---| +| `memory_layer/memory_extractor/agent_skill_extractor.py:325` | 3 | **0** | `None` | +| `memory_layer/memory_extractor/agent_case_extractor.py:371` | 2 | **0** | `None` | +| `memory_layer/memory_extractor/agent_case_extractor.py:416` | 2 | **0** | `None` | +| `memory_layer/cluster_manager/manager.py:636` | 3 | **0** | `None` | +| `memory_layer/profile_manager/manager.py:178` | `max_retries` | `0.5 * (attempt+1)` (**linear**) | — | + +**Problem analysis**: +1. **Type B has no sleep at all**: LLMs are high-latency (seconds) and billed per call. **Three back-to-back requests** = real money burned + a high chance of being stuck on the same prompt → same wrong result (LLMs are largely deterministic). +2. **Should feed the error back into the prompt**: when the format is wrong, don't retry the same prompt — append "previous reply + error description" so the LLM can correct itself. +3. **Silent `return None` on failure**: callers receive `None` and read it as "no content"; they can't tell "the model genuinely produced nothing" from "the model malformed 3 times in a row". +4. **Type B stacks on top of Type A**: A already retries 3-5 times at the HTTP layer, B retries 3 more at the business layer → a single business call can fire up to **3 × 3 = 9 LLM calls**. There is no total-budget control. +5. **Backoff strategies conflict**: `profile_manager` uses linear `0.5*(attempt+1)` (0.5 s, 1 s, 1.5 s …), inconsistent with the exponential backoff used elsewhere. Two engineers wrote two policies. + +### 4.4 Type D: Two duplicated `RetryConfig` classes + +| Location | Purpose | +|---|---| +| `core/longjob/interfaces.py:183` | `class RetryConfig` for the longjob consumer | +| `core/asynctasks/task_manager.py:54` | `class RetryConfig` for the ARQ `@task` decorator | + +The two classes have **heavily overlapping fields** (`max_retries`, `exponential_backoff`, …) but are defined independently. + +**Impact**: every bug fix touches two places; new retry-policy fields must be added twice; test coverage is split. + +**Fix**: extract a single `core/retry/config.py` and have both call sites reference it. + +### 4.5 Type E: Service-Level "Circuit Breaker" (crude implementation) + +`HybridVectorizeService` / `HybridRerankService`: + +```python +self._primary_failure_count: int = field(default=0, init=False, repr=False) +self.max_primary_failures: int = 3 +``` + +Logic: after N consecutive failures on the primary provider, switch to the fallback. + +**Problems** (also flagged in a previous audit): +1. `_primary_failure_count` is an **instance attribute with no lock**. +2. When multiple coroutines call `execute_with_fallback` concurrently there is a race condition — the counter can under- or over-count. +3. **No half-open state**: once you switch to fallback, the primary is never probed again. +4. **No timeout / reset window**: a single morning hiccup → the fallback is used all day (even after the primary has recovered). + +This isn't really "retry" — it's a crude circuit breaker. Use a library such as `aiobreaker` or `purgatory`. + +### 4.6 Type F: Loop-local try/continue ("skip on failure") + +| File | Behaviour | Emits metric? | +|---|---|---| +| `core/cache/redis_cache_queue/redis_length_cache_manager.py:480-487` | Batch processing skips one failing item and continues | ❌ only `logger.warning` | +| `core/cache/redis_cache_queue/redis_windows_cache_manager.py:465-472` | Same | ❌ | +| `agentic_layer/retrieval_utils.py:233-241` | Skip on vector-dot failure | ❌ | + +**Problem**: callers receive a partial result with **no way to tell "all succeeded" from "50% failed"**. Add a `partial_failure_count` metric. + +### 4.7 Worked example: stacked retry amplification + +Picture a `memorize()` call that triggers LLM extraction: + +``` +controller.add_memories() +└─ memory_manager.memorize() + └─ AgentSkillExtractor._call_llm() ← Type B: for attempt in range(3), no sleep + └─ LLMProvider.generate() + └─ AnthropicAdapter.chat() ← Type A: for attempt in range(max_retries) with 2**attempt + └─ httpx.post() ← httpx has no built-in retry +``` + +Worst-case call count: +- Type A `max_retries=3` → 1 + 2 + 4 = 7 s of waiting, 3 calls +- Type B `retries=3`, no sleep → 3 outer attempts +- **Final: 3 × 3 = 9 HTTP requests**, longest wait ~21 s (each outer attempt waits up to 7 s) + +And: +- On total failure, **only `None` is returned** → controller sees `0 memories extracted` +- The client sees HTTP 200 and assumes "no new memories" +- Real story: 9 retries all failed, and the bill has been charged 9 × token cost + +### 4.8 Retry-Refactor Recommendation: Standardise on `tenacity` + +**One decorator replaces 6 copy-pasted blocks**: + +```python +from tenacity import ( + retry, retry_if_exception_type, stop_after_attempt, + wait_exponential_jitter, before_sleep_log, +) + +@retry( + retry=retry_if_exception_type((httpx.HTTPStatusError, asyncio.TimeoutError)), + stop=stop_after_attempt(3), + wait=wait_exponential_jitter(initial=1, max=10, jitter=2), + reraise=True, + before_sleep=before_sleep_log(logger, logging.WARNING), +) +async def _call_llm(...): + response = await client.post(...) + response.raise_for_status() # ← 5xx raises and is retried; 4xx raises and is filtered out by retry_if_exception_type + return response.json() +``` + +Benefits: +- 6 boilerplate copies → 1 decorator +- Jitter baked in (solves thundering herd) +- `retry_if_exception_type` gives precise control (4xx isn't retried, saving quota) +- `RetryError` on final failure carries the full attempt history (cause chain preserved) +- `before_sleep_log` emits a log entry per retry — no need for manual `logger.warning` + +**Refactor for Type B LLM output-format retry**: + +```python +# Anti-pattern: hard-retry the same prompt +for attempt in range(3): + resp = await llm.generate(prompt) + if valid(resp): return resp +return None + +# Correct: feed the error signal back into the prompt +attempt_history = [] +for attempt in range(3): + enriched_prompt = prompt + format_correction_hints(attempt_history) + resp = await llm.generate(enriched_prompt) + if validation := validate(resp): + return resp + attempt_history.append((resp, validation.errors)) +raise InvalidLLMOutputError(attempt_history) +``` + +Or even cleaner — **don't retry at all**, just raise and let the upper layer decide (re-run with a stronger model, downgrade, hand off to a human): + +```python +resp = await llm.generate(prompt) +data = parse_json_response(resp) +if not (data and isinstance(data.get("operations"), list)): + raise InvalidLLMOutputError(resp) +return data +``` + +### 4.9 Retry-Refactor Action List + +| Priority | Item | Effort | Risk | +|---|---|---|---| +| P0 | Merge the two `RetryConfig` classes | 1 PR | Low | +| P0 | Type A: wrap the 6 HTTP retries with one tenacity decorator | 1 week | Medium (needs tests) | +| P1 | Type B: add sleep + jitter to the 5 LLM-format retries, or switch to "feedback retry" | Medium | Medium | +| P1 | Type E: introduce `aiobreaker` or lock the fallback counters | 1 PR | Medium | +| P2 | Type F: add a `partial_failure_count` metric to loop-continue paths | Small | Low | +| P3 | Global retry budget (total-time / total-call caps) | Design | Large | + +--- + +## 5. Performance Impact + +### 5.1 The `try` block itself ≈ free + +CPython 3.11 shipped **"zero-cost exceptions"**: +- Old mechanism: entering `try:` emitted a `SETUP_FINALLY` opcode and ran a few instructions per execution +- New mechanism: the exception-handling table is moved out of bytecode into a side table; the happy path executes **zero exception-related instructions** +- Only the unwinding path runs when an exception is **actually raised** + +This project targets `requires-python = ">=3.12,<3.13"`, so **all 781 `try` blocks add zero overhead on the happy path**. + +### 5.2 Cost when an exception actually fires + +Each raise/catch costs roughly 10–100 μs (depending on stack depth), spent on: +- Building the traceback object +- Walking the stack frames +- Formatting the exception message + +But the project's most expensive operations are: +- LLM API calls: **hundreds of ms to several seconds** +- Milvus vector search: **tens to hundreds of ms** +- Elasticsearch queries: **tens of ms** + +Tens of microseconds of exception cost is **entirely swallowed by network latency**. + +### 5.3 What actually burns CPU is the side-work inside `except` + +| Operation | Cost per call | Occurrences | Cumulative impact | +|---|---|---|---| +| `traceback.print_exc()` | ms-level | 20 | Medium | +| f-string in `logger.debug` | μs per call | 247 | High (every request × N) | +| `traceback.format_exc()` + string concatenation | ms-level | Scattered | Medium | + +### 5.4 Hidden performance cost: side effects on the error path + +**Example 1**: Retrying immediately with no backoff after an exception +- When the backend is rate-limited (OpenRouter, DeepInfra), no-backoff retries get you banned faster. +- Use `tenacity` or explicit exponential backoff. + +**Example 2**: Falling back to a full scan / full recompute on exception +- When `preprocess_conv_request` fails, it returns the original request; downstream code may then run a much larger batch. +- The "recovery" is more expensive than the failure. + +### 5.5 Performance Verdict + +**Density itself is not a performance problem.** What needs fixing is: +1. f-strings in logger calls (247 cases) — machine-detectable via ruff `G004` +2. `traceback.print_exc()` (20 cases) — replace by hand or via ruff `T201` +3. "Recovery" logic on error paths (no-backoff retries, full fallbacks) — design-level review + +--- + +## 6. Correctness Impact (the part that really matters) + +### 6.1 Signal-Erosion Chain + +``` + Original exception (precise, locatable) + ↓ Caught by catch-all + logger.error("failed") + ↓ return None / [] / 0 + Upper layer sees "empty result" + ↓ Treats it as "no data" + API returns 200 + "accumulated" + ↓ Client + "Everything looks fine" +``` + +Every layer of `except Exception: log; return default` drops a level of signal. By the time the request reaches the API boundary you can no longer tell: +- Genuinely no data +- Interface failure +- Programming bug +- Configuration error + +### 6.2 Real Historical Case + +From the earlier audit, the `milvus_start` possibly-unbound bug: +- Inside `memory_manager.get_vector_search_results`, `milvus_start` was assigned only at line 729 (inside the try) +- The `except` block at line 782 referenced `milvus_start` +- If an exception fired before line 729 → `UnboundLocalError` was thrown inside the `except` +- **The original ES/Milvus error was masked**; what the logger saw was "variable referenced before assignment" + +This is the classic side-effect of catch-all: a locatable failure becomes a second-order puzzle. + +### 6.3 Observability Loss + +71% of excepts being `except Exception` means: +- The Prometheus error-type metric (`record_retrieve_error`) can only report `'unknown_error'` +- Alerts can only fire on overall error rate, not on specific error categories +- Sentry / error-aggregation tools cannot get a specific exception class — every issue collapses into one bucket + +--- + +## 7. Refactoring Guide + +### 7.1 Decision Tree: do you need a try? + +``` +Can this line raise? +├── No → don't wrap it +├── Yes, raises a ProgrammingError (NameError/TypeError, etc.) +│ → don't wrap it; let CI + type-checking + crash-on-deploy surface it +├── Yes, raises a ResourceError (DBError/HTTPError/Timeout, etc.) +│ ├── I can recover → try only that line; except a specific type; carry on +│ ├── I can degrade → try only that line; except a specific type; emit a degradation metric +│ ├── I cannot recover → don't wrap it; let it propagate +│ └── It's a fire-and-forget background task → wrap the whole worker, but emit a failure metric +``` + +### 7.2 Replacement Cookbook + +| Current pattern | Recommended replacement | +|---|---| +| `try: ...; except Exception as e: logger.error(e); raise` | `try: ...; except SpecificError as e: raise NewError(...) from e` | +| `try: api_call() except Exception: time.sleep(1); retry` | `@tenacity.retry(stop=stop_after_attempt(3), wait=wait_exponential())` | +| `try: ...; except Exception: pass` | Don't. Let it crash. If you really don't care: `contextlib.suppress(SpecificError)` | +| `except Exception: return None` (business layer) | Don't catch; let the controller turn it into 5xx | +| `traceback.print_exc()` | `logger.error("...", exc_info=True)` | +| `logger.error(f"...{var}")` | `logger.error("...%s", var)` | + +### 7.3 Refactor Priority + +Ordered by ROI, **Phase 1 is mechanical fixes that don't change behaviour**: + +| Priority | Item | Volume | Approach | Risk | +|---|---|---|---|---| +| P0 | `traceback.print_exc()` → `exc_info=True` | 20 | Semi-automatic (grep + sed) | Very low | +| P0 | f-string in logger → `%`-style | 247 | ruff `G004` + manual review | Low | +| P1 | Split the top-5 oversize try blocks | 5 files | Manual | Medium | +| P1 | Convert biz_layer catch-all to upward propagation | ~15 sites | Manual + add boundary except in controllers | Medium-high | +| P2 | `except Exception` → specific exceptions | ~555 | Gradual; a few per PR | High | +| P3 | Replace hand-written retries with `tenacity` | Scattered | Rewrite | Medium | + +### 7.4 Worked Refactor: `mem_memorize.memorize()` + +**Current** (simplified): +```python +async def memorize(request) -> int: + try: + for memcell in memcells: + await process_memory_extraction(memcell, ...) + return memories_count + except Exception as e: + logger.error(f"failed: {e}") + return 0 +``` + +**After refactor**: +```python +async def memorize(request) -> int: + # A single failing memcell should not block the others + processed = 0 + failures = [] + for memcell in memcells: + try: + await process_memory_extraction(memcell, ...) + processed += 1 + except (MilvusError, ESError, MongoError) as e: + # Known storage-layer failure: record, emit metric, continue + failures.append((memcell.event_id, e)) + record_memorize_error(stage='persist', error_type=type(e).__name__) + logger.error("memcell %s failed: %s", memcell.event_id, e, exc_info=True) + # Don't swallow LLMError, NameError, ValidationError — let them propagate + + if failures and len(failures) == len(memcells): + # All failed → real outage, surface as 5xx + raise MemorizeFailure(failures) + return processed +``` + +Benefits: +- Partial failure becomes visible (metric + log) +- Total failure raises a 5xx, so the client knows +- Programming bugs (NameError, etc.) are no longer swallowed + +--- + +## 8. Tooling to Prevent Re-Regression + +### 8.1 Recommended ruff Rules + +Add these to the `select` list in `[tool.ruff.lint]` of `pyproject.toml`: + +```toml +select = [ + # ... existing rules ... + "BLE", # flake8-blind-except — catches bare `except:` and `except Exception:` without a logger + "G", # flake8-logging-format — catches f-strings inside logger calls + "TRY", # tryceratops — the full exception-handling rule family +] + +# In Phase 1 enable per-directory to avoid 500+ violations exploding at once +[tool.ruff.lint.per-file-ignores] +"src/biz_layer/**/*.py" = ["BLE001", "TRY"] # temporarily exempt this directory; tighten over time +``` + +**Key rules**: + +| Rule | What it catches | Notes | +|---|---|---| +| `BLE001` | `except Exception` or bare `except` | Enabling this alone surfaces 555 violations | +| `G004` | f-string inside a logger call | 247 occurrences | +| `G201` | `logger.error(...)` that should pass `exc_info=True` | | +| `TRY003` | Raising `Exception(f"...{var}")` with an inline message | Encourages defining custom exception classes | +| `TRY200` | A `raise` inside `except` missing `from e` | Loses the cause chain | +| `TRY300` | `return` inside `try` that should move to `else` | Keeps the try scope narrow | +| `TRY400` | `logger.error` in an except where `logger.exception` is appropriate | | + +### 8.2 Metrics to Add + +During and after the refactor, add these metrics to observe progress: + +- `except_handled_total{module, exception_type}` — counts the exceptions actually caught +- `silent_failure_total{module, function}` — incremented when an except returns a default value instead of raising +- `error_type_unknown_ratio` — the share of `'unknown_error'` labels in total errors (target < 5%) + +### 8.3 Codified Conventions + +Suggested additions to `docs/dev_docs/development_standards.md`: + +> **Three iron rules of exception handling** +> 1. **The business layer (`biz_layer`) must not catch `Exception`.** Let exceptions propagate to the controller. +> 2. **Boundary code (`infra_layer`, external API calls in `agentic_layer`) must catch specific exception types** and use `raise ... from e` to preserve the cause chain. +> 3. **Never use `traceback.print_exc()` in an `except` block.** Always use `logger.exception(msg)` or `logger.error(msg, exc_info=True)`. + +--- + +## 9. Action Plan (organised by sprint) + +### Sprint 1 (1 day, zero risk) +- [ ] Replace all `traceback.print_exc()` with `exc_info=True` (20 sites) +- [ ] Enable `ruff G004` and run `--fix` to clean up f-string-in-logger (247 sites) +- [ ] Enable `ruff BLE001` and add it to `per-file-ignores` as the baseline +- [ ] Merge the two `RetryConfig` classes (§4.4) → single source + +### Sprint 2 (2-3 days, medium risk) +- [ ] Split the top-5 oversize try blocks (one independent PR + test coverage each) +- [ ] Refactor `mem_memorize.memorize()` along the lines of §7.4 +- [ ] Add boundary except in controllers to catch the specific exceptions thrown by the business layer +- [ ] Add sleep + jitter to Type B LLM output-format retries (§4.3), or rewrite them as "feedback retries" + +### Sprint 3 (1 week, gradual) +- [ ] Introduce a custom exception hierarchy (`MemorizeError`, `RetrieveError`, `ExtractionError`, `InvalidLLMOutputError`) +- [ ] Boundary code (repositories, external API clients) switches `except Exception` to specific types +- [ ] Each PR shrinks the scope of ruff `BLE001` per-file-ignores +- [ ] Adopt `tenacity` and consolidate the 6 Type A HTTP retries (§4.2) + +### Sprint 4 (long term) +- [ ] Replace the crude Type E fallback in `HybridVectorizeService` / `HybridRerankService` with `aiobreaker` (§4.5) +- [ ] Add `partial_failure_count` metrics to Type F loop-continue paths (§4.6) +- [ ] Global retry budget (caps on total time / total call count) — design + implementation +- [ ] Add exception handling to the PR review checklist +- [ ] Run mutation testing to verify that `except` block behaviour really has coverage + +--- + +## Appendix A: Commands to Reproduce the Numbers in This Report + +```bash +# Total try blocks +grep -rn "^\s*try:" src/ --include="*.py" | wc -l + +# Total function definitions +grep -rn "^\s*\(async \)\?def " src/ --include="*.py" | wc -l + +# Distribution of except clause shapes +grep -rh "^\s*except" src/ --include="*.py" \ + | sed -E 's/.*(except[^:]*).*/\1/' \ + | sort | uniq -c | sort -rn + +# f-string in logger +grep -rn 'logger\.\(debug\|info\|warning\|error\|critical\)(f"' \ + src/ --include="*.py" | wc -l + +# Remaining traceback.print_exc() +grep -rn "traceback.print_exc()" src/ --include="*.py" | wc -l +``` + +## Appendix B: References + +- [PEP 657 — Include Fine-Grained Error Locations in Tracebacks](https://peps.python.org/pep-0657/) +- [What's New in Python 3.11: Zero-cost Exceptions](https://docs.python.org/3/whatsnew/3.11.html#cpython-bytecode-changes) +- [Ruff Tryceratops rules](https://docs.astral.sh/ruff/rules/#tryceratops-try) +- [tenacity documentation](https://tenacity.readthedocs.io/) +- Robert C. Martin, *Clean Code*, Ch. 7 "Error Handling" From a60168eb47eecd22111242cf5bf3b3a96d5a3aab Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:15:08 +0800 Subject: [PATCH 13/24] feat(EverCore): add /livez and /readyz health endpoints K8s-style liveness vs readiness separation, per the observability roadmap (Phase 1, T1.1). - /livez: process heartbeat, always 200. Suitable for liveness probes; a failure means the process should be restarted. - /readyz: runs four downstream probes (Redis, MongoDB, Elasticsearch, Milvus) in parallel under a 2 s timeout. Returns 200 only when all succeed; 503 with a per-dependency breakdown otherwise. Suitable for readiness probes and load-balancer drain checks. - /health: legacy alias for /livez, preserved for backward compatibility. Each probe also publishes the evercore_dependency_healthy{name="..."} Prometheus gauge so alerts can fire without polling /readyz externally. Probe definitions live in a sibling probes.py module; adding a new downstream is one async function plus an entry in default_probes(). prometheus_middleware skips /livez and /readyz so probe traffic does not dominate request-rate dashboards. Smoke-tested locally: happy path returns 200 with all four deps healthy; pointing REDIS_PORT at a dead port correctly drives /readyz to 503 with redis=False detail, /livez stays 200, and the gauge for redis drops to 0 while others remain 1. --- .../core/middleware/prometheus_middleware.py | 14 +- .../input/api/health/health_controller.py | 118 +++++++++----- .../adapters/input/api/health/probes.py | 153 ++++++++++++++++++ 3 files changed, 240 insertions(+), 45 deletions(-) create mode 100644 methods/EverCore/src/infra_layer/adapters/input/api/health/probes.py diff --git a/methods/EverCore/src/core/middleware/prometheus_middleware.py b/methods/EverCore/src/core/middleware/prometheus_middleware.py index ca9845d6..7af94465 100644 --- a/methods/EverCore/src/core/middleware/prometheus_middleware.py +++ b/methods/EverCore/src/core/middleware/prometheus_middleware.py @@ -137,8 +137,18 @@ class PrometheusMiddleware(BaseHTTPMiddleware): app.add_middleware(PrometheusMiddleware) """ - # Paths to skip metrics collection - SKIP_PATHS = {'/metrics', '/health', '/healthz', '/ready', '/favicon.ico'} + # Paths to skip metrics collection. + # /livez and /readyz are K8s probe endpoints — including them would let + # probe traffic dominate request-rate dashboards. + SKIP_PATHS = { + '/metrics', + '/health', + '/healthz', + '/ready', + '/livez', + '/readyz', + '/favicon.ico', + } async def dispatch(self, request: Request, call_next: Callable) -> Response: # Skip metrics for certain paths diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/health/health_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/health/health_controller.py index 5f98bb0c..b568a539 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/health/health_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/health/health_controller.py @@ -1,64 +1,96 @@ -""" -Health check controller +"""Health-check endpoints. + +The controller exposes three K8s-style endpoints so liveness and readiness +can be distinguished cleanly: + +- ``/livez`` — process heartbeat. Always 200. Use this for K8s liveness + probes. A failure means the process should be restarted. +- ``/readyz`` — process plus downstream connectivity. 200 if every probe + in :mod:`probes.default_probes` succeeds, 503 otherwise. + Use this for K8s readiness probes and load-balancer + drain checks. A failure means traffic should be steered + away until dependencies recover. +- ``/health`` — legacy alias kept for backward compatibility. Behaves + like ``/livez``. -Provides system health status check interface +The per-dependency status is also published as the +``evercore_dependency_healthy{name="..."}`` Prometheus gauge, so alerts +can fire without polling these endpoints from outside. """ +from typing import Any, Dict + +from fastapi.responses import JSONResponse + from common_utils.datetime_utils import get_now_with_timezone -from typing import Dict, Any +from core.di.decorators import component from core.interface.controller.base_controller import BaseController, get from core.observation.logger import get_logger -from core.di.decorators import component + +from .probes import default_probes, run_all logger = get_logger(__name__) @component(name="healthController") class HealthController(BaseController): - """ - Health check controller + """Liveness, readiness, and legacy health endpoints.""" - Provides system health status check functionality - """ + # Probe timeout: short enough that K8s probe budgets aren't blown, + # long enough that a healthy backend on a transient hiccup still wins. + READINESS_PROBE_TIMEOUT_S = 2.0 - def __init__(self): + def __init__(self) -> None: super().__init__( - prefix="/health", + prefix="", # endpoints live at the application root tags=["Health"], - default_auth="none", # Health check does not require authentication + default_auth="none", # probes must not require auth ) - @get("", summary="Health check", description="Check system health status") - def health_check(self) -> Dict[str, Any]: - """ - Health check interface + @get("/livez", summary="Liveness probe", description="Process is alive (no downstream checks).") + def livez(self) -> Dict[str, Any]: + """Cheap heartbeat. Always returns 200 if the event loop is running.""" + return { + "status": "alive", + "timestamp": get_now_with_timezone().isoformat(), + } - Returns: - Dict[str, Any]: Health status information + @get( + "/readyz", + summary="Readiness probe", + description="200 only when every downstream dependency is reachable.", + ) + async def readyz(self) -> JSONResponse: + """Run every dependency probe and aggregate the verdict. - Raises: - HTTPException: Throws 500 error when system is unhealthy + Returns 200 with the per-dependency breakdown when all probes pass. + Returns 503 with the *same* JSON shape when any probe fails, so + clients can parse a single schema for both outcomes. + + We return a ``JSONResponse`` directly rather than raising + ``HTTPException``; the project's global exception handler reshapes + HTTPException detail through ``ErrorApiResponse`` (which requires + ``message: str``), which would corrupt the structured payload. """ - try: - # Log health check request - logger.debug("Health check request") - - # Return simple health status - return { - "status": "healthy", - "timestamp": get_now_with_timezone().isoformat(), - "message": "System running normally", - } - except Exception as e: - logger.error(f"Health check failed: {str(e)}") - # Throw 500 error when exception occurs - from fastapi import HTTPException - - raise HTTPException( - status_code=500, - detail={ - "status": "unhealthy", - "timestamp": get_now_with_timezone().isoformat(), - "message": f"System check exception: {str(e)}", - }, - ) + results = await run_all( + default_probes(), timeout=self.READINESS_PROBE_TIMEOUT_S + ) + all_healthy = all(r.healthy for r in results) + payload: Dict[str, Any] = { + "status": "ready" if all_healthy else "not_ready", + "timestamp": get_now_with_timezone().isoformat(), + "dependencies": [r.to_dict() for r in results], + } + # 503 Service Unavailable is the standard "alive but not serving + # traffic" signal that K8s readiness probes expect. + status_code = 200 if all_healthy else 503 + return JSONResponse(content=payload, status_code=status_code) + + @get( + "/health", + summary="Legacy health check", + description="Deprecated alias for /livez. Kept for backward compatibility.", + ) + def health(self) -> Dict[str, Any]: + """Alias for ``/livez`` so existing clients keep working.""" + return self.livez() diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/health/probes.py b/methods/EverCore/src/infra_layer/adapters/input/api/health/probes.py new file mode 100644 index 00000000..45b46cc4 --- /dev/null +++ b/methods/EverCore/src/infra_layer/adapters/input/api/health/probes.py @@ -0,0 +1,153 @@ +"""Readiness probes for downstream dependencies. + +Each probe is a short async check that returns a ProbeResult. They are +executed in parallel by the health controller under a per-probe timeout +so a slow or hung downstream cannot block /readyz indefinitely. + +Adding a new dependency is a matter of writing one async function that +raises on failure and registering it in `default_probes()`. +""" + +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass +from typing import Awaitable, Callable, List, Optional + +from prometheus_client import Gauge as PrometheusGauge + +from core.di.utils import get_bean_by_type +from core.observation.logger import get_logger +from core.observation.metrics.registry import get_metrics_registry + +logger = get_logger(__name__) + + +# Per-dependency health gauge. 1 = healthy, 0 = unhealthy. +# Scraped by Prometheus; can drive alerts independent of `/readyz` polling. +# +# We bypass the BaseGauge wrapper (which is designed for auto-refresh) and +# bind a raw prometheus_client.Gauge to the project registry, so probe code +# can call set() directly. +_dependency_healthy = PrometheusGauge( + name="evercore_dependency_healthy", + documentation="1 if the named downstream is reachable, 0 otherwise.", + labelnames=["name"], + registry=get_metrics_registry(), +) + + +@dataclass +class ProbeResult: + """Outcome of a single readiness probe.""" + + name: str + healthy: bool + detail: Optional[str] + latency_ms: float + + def to_dict(self) -> dict: + return { + "name": self.name, + "healthy": self.healthy, + "detail": self.detail, + "latency_ms": round(self.latency_ms, 2), + } + + +Probe = Callable[[], Awaitable[None]] +"""A probe is an async callable that returns normally on success and raises +any exception on failure. It must not return a value.""" + + +async def run_probe(name: str, check: Probe, timeout: float = 2.0) -> ProbeResult: + """Run one probe under a timeout and capture its outcome. + + The probe contract is: succeed silently or raise. Timeouts and any + other exception are converted into an unhealthy ProbeResult so the + caller never has to wrap individual probes in try/except. + """ + start = time.perf_counter() + detail: Optional[str] = None + healthy = False + try: + await asyncio.wait_for(check(), timeout=timeout) + healthy = True + except asyncio.TimeoutError: + detail = f"timeout after {timeout}s" + except Exception as exc: # noqa: BLE001 - boundary code: convert to result + detail = f"{type(exc).__name__}: {exc}" + latency_ms = (time.perf_counter() - start) * 1000 + + _dependency_healthy.labels(name=name).set(1.0 if healthy else 0.0) + if not healthy: + logger.warning( + "Readiness probe failed: name=%s detail=%s latency_ms=%.2f", + name, + detail, + latency_ms, + ) + return ProbeResult(name=name, healthy=healthy, detail=detail, latency_ms=latency_ms) + + +async def run_all(probes: List[tuple[str, Probe]], timeout: float = 2.0) -> List[ProbeResult]: + """Run every probe in parallel and return all results in input order.""" + tasks = [run_probe(name, check, timeout=timeout) for name, check in probes] + return await asyncio.gather(*tasks) + + +# --------------------------------------------------------------------------- +# Individual probes +# --------------------------------------------------------------------------- + + +async def _probe_redis() -> None: + from core.component.redis_provider import RedisProvider + + provider = get_bean_by_type(RedisProvider) + client = await provider.get_client() + await client.ping() + + +async def _probe_mongodb() -> None: + from core.component.mongodb_client_factory import MongoDBClientFactory + + factory = get_bean_by_type(MongoDBClientFactory) + wrapper = await factory.get_default_client() + await wrapper.client.admin.command("ping") + + +async def _probe_elasticsearch() -> None: + from core.component.elasticsearch_client_factory import ElasticsearchClientFactory + + factory = get_bean_by_type(ElasticsearchClientFactory) + # The factory raises NotImplementedError on get_default_client; the + # canonical accessor is register_default_client, which is idempotent + # and returns the cached wrapper on subsequent calls. + wrapper = await factory.register_default_client() + if not await wrapper.async_client.ping(): + raise RuntimeError("elasticsearch ping returned False") + + +async def _probe_milvus() -> None: + from core.component.milvus_client_factory import MilvusClientFactory + + factory = get_bean_by_type(MilvusClientFactory) + client = factory.get_default_client() + # Milvus client is synchronous; run on a thread to keep the event loop free. + await asyncio.to_thread(client.list_collections) + + +def default_probes() -> List[tuple[str, Probe]]: + """The standard set of downstream probes for /readyz. + + Add new probes here. The order shapes the response payload but not the + overall healthy/unhealthy verdict — every probe runs unconditionally. + """ + return [ + ("redis", _probe_redis), + ("mongodb", _probe_mongodb), + ("elasticsearch", _probe_elasticsearch), + ("milvus", _probe_milvus), + ] From 1ad942c8874668a17b2354b534c270b088affcbd Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:17:49 +0800 Subject: [PATCH 14/24] feat(EverCore): unify error_type metric labels via classify_exception Per Phase 1, T1.3 of the observability roadmap. Before this change, four independent classifiers (memorize/retrieve/rerank/vectorize) each did string-matching on str(error).lower() and fell through to the literal "unknown" (or "error") bucket. Cross-service alerts could not group failures the same way, and any unfamiliar exception class was invisible behind the catch-all label. The new core/observation/error_classification.py is the single source of truth: it still emits the existing semantic labels (timeout, rate_limit, validation_error, connection_error, not_found) when those match, and otherwise returns the exception class name in snake_case. The four call sites now delegate to it. Effect on metrics: - error_type='unknown' / error_type='error' goes away entirely. - A previously-anonymous KeyError or pydantic.ValidationError now shows up as error_type='key_error' / 'validation_error' and can be alerted on. - Vectorize-specific failures previously bucketed as 'api_error' now surface as 'vectorize_error', which is more actionable. Unit-tested with 9 representative exception shapes; ruff + compile clean. No call-site behaviour change beyond the label string. --- .../src/agentic_layer/memory_manager.py | 21 +++--- .../agentic_layer/metrics/memorize_metrics.py | 18 ++--- .../src/agentic_layer/rerank_service.py | 25 +++---- .../src/agentic_layer/vectorize_service.py | 22 +++--- .../core/observation/error_classification.py | 75 +++++++++++++++++++ 5 files changed, 111 insertions(+), 50 deletions(-) create mode 100644 methods/EverCore/src/core/observation/error_classification.py diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index 1118c0fb..96f05a6e 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -911,18 +911,15 @@ async def _search_hybrid( return reranked def _classify_retrieve_error(self, error: Exception) -> str: - """Classify error type for metrics""" - error_str = str(error).lower() - if 'timeout' in error_str or 'timed out' in error_str: - return 'timeout' - elif 'connection' in error_str or 'connect' in error_str: - return 'connection_error' - elif 'not found' in error_str or 'notfound' in error_str: - return 'not_found' - elif 'validation' in error_str or 'invalid' in error_str: - return 'validation_error' - else: - return 'unknown' + """Classify error type for metrics. + + Delegates to :func:`classify_exception` so retrieve, rerank, and + vectorize metrics share a single taxonomy. Unknown errors now + report their concrete class name instead of ``"unknown"``. + """ + from core.observation.error_classification import classify_exception + + return classify_exception(error) async def _to_response( self, hits: List[Dict], req: 'RetrieveMemRequest' diff --git a/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py b/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py index 5915ceff..64b11480 100644 --- a/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py @@ -376,18 +376,16 @@ def record_memorize_message( def classify_memorize_error(error: Exception) -> str: - """ - Classify error type for metrics - - Args: - error: Exception instance + """Classify ``error`` into a stable Prometheus label. - Returns: - Error type string for metrics label + Delegates to the shared :func:`classify_exception` so the memorize + pipeline uses the same taxonomy as retrieve / rerank / vectorize + metrics. Unknown exception classes surface their snake-cased class + name instead of the previous opaque ``"error"`` bucket. """ - # TODO: Add detailed error classification based on actual scenarios - _ = error # Placeholder for future use - return 'error' + from core.observation.error_classification import classify_exception + + return classify_exception(error) def record_boundary_detection( diff --git a/methods/EverCore/src/agentic_layer/rerank_service.py b/methods/EverCore/src/agentic_layer/rerank_service.py index 404ad570..251cff50 100644 --- a/methods/EverCore/src/agentic_layer/rerank_service.py +++ b/methods/EverCore/src/agentic_layer/rerank_service.py @@ -491,22 +491,15 @@ async def execute_with_fallback( ) def _classify_error(self, error: Exception) -> str: - """Classify error type for metrics""" - error_str = str(error).lower() - error_type = type(error).__name__.lower() - - if 'timeout' in error_str or 'timeout' in error_type: - return 'timeout' - elif 'rate' in error_str and 'limit' in error_str: - return 'rate_limit' - elif 'validation' in error_str or 'invalid' in error_str: - return 'validation_error' - elif 'connection' in error_str or 'connect' in error_type: - return 'connection_error' - elif 'api' in error_str or 'http' in error_str: - return 'api_error' - else: - return 'unknown' + """Classify error type for metrics. + + Delegates to :func:`classify_exception` for a project-wide taxonomy. + Unknown errors surface their concrete class name instead of the + previous opaque ``"unknown"`` bucket. + """ + from core.observation.error_classification import classify_exception + + return classify_exception(error) def get_failure_count(self) -> int: """Get current primary service failure count""" diff --git a/methods/EverCore/src/agentic_layer/vectorize_service.py b/methods/EverCore/src/agentic_layer/vectorize_service.py index 7b94e4b4..d0a3fee3 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_service.py +++ b/methods/EverCore/src/agentic_layer/vectorize_service.py @@ -487,18 +487,16 @@ async def execute_with_fallback( ) def _classify_error(self, error: Exception) -> str: - """Classify error type for metrics""" - error_str = str(error).lower() - if 'timeout' in error_str or 'timed out' in error_str: - return 'timeout' - elif 'rate' in error_str and 'limit' in error_str: - return 'rate_limit' - elif 'validation' in error_str or 'invalid' in error_str: - return 'validation_error' - elif isinstance(error, VectorizeError): - return 'api_error' - else: - return 'unknown' + """Classify error type for metrics. + + Delegates to :func:`classify_exception` so vectorize and rerank + share the same taxonomy. ``VectorizeError`` falls through to + ``vectorize_error`` via the class-name path, which is more + actionable than the previous flat ``"api_error"``. + """ + from core.observation.error_classification import classify_exception + + return classify_exception(error) def get_failure_count(self) -> int: """Get current primary service failure count""" diff --git a/methods/EverCore/src/core/observation/error_classification.py b/methods/EverCore/src/core/observation/error_classification.py new file mode 100644 index 00000000..af4b712f --- /dev/null +++ b/methods/EverCore/src/core/observation/error_classification.py @@ -0,0 +1,75 @@ +"""Shared exception-to-label classifier for Prometheus metrics. + +Why this module exists +---------------------- +Before this helper landed, every service had its own ``_classify_error`` +method using string-matching against ``str(error)``, and the default +branch returned the hardcoded literal ``"unknown"``. That blew up +observability in two ways: + +* Multiple sites returned different strings for the same exception class + (``"connection_error"`` vs ``"api_error"``), so cross-service alerting + was impossible. +* The default ``"unknown"`` bucket swallowed every unfamiliar exception, + so Grafana's error-type breakdown was useless once anything new + appeared. + +The function here is the single source of truth. Specific semantic +buckets (``timeout``, ``rate_limit``, ``validation_error``, +``connection_error``) are still returned when recognised, and *anything +else* falls through to the exception class name converted to +``snake_case`` so the label is always a concrete, queryable signal. +""" + +from __future__ import annotations + +import re + + +_CAMEL_TO_SNAKE_FIRST = re.compile(r"(.)([A-Z][a-z]+)") +_CAMEL_TO_SNAKE_REST = re.compile(r"([a-z0-9])([A-Z])") + + +def _to_snake_case(name: str) -> str: + """``HTTPSConnectionError`` → ``https_connection_error``.""" + s1 = _CAMEL_TO_SNAKE_FIRST.sub(r"\1_\2", name) + return _CAMEL_TO_SNAKE_REST.sub(r"\1_\2", s1).lower() + + +def classify_exception(error: BaseException) -> str: + """Return a stable, Prometheus-friendly label for ``error``. + + Lookup order: + + 1. Known semantic categories matched against the exception class name + and message — keeps existing dashboards working. + 2. The exception class name converted to ``snake_case`` for anything + else. This guarantees the label is never the opaque ``"unknown"``. + """ + cls_name = type(error).__name__ + cls_lower = cls_name.lower() + msg_lower = str(error).lower() + + # Semantic categories first so existing dashboards keep grouping + # well-known failure modes. + if "timeout" in cls_lower or "timeout" in msg_lower or "timed out" in msg_lower: + return "timeout" + if ("rate" in msg_lower and "limit" in msg_lower) or "ratelimit" in cls_lower: + return "rate_limit" + if ( + "validation" in cls_lower + or "validation" in msg_lower + or "invalid" in msg_lower + ): + return "validation_error" + if ( + "connection" in cls_lower + or "connect" in cls_lower + or "connection" in msg_lower + ): + return "connection_error" + if "notfound" in cls_lower or "not found" in msg_lower: + return "not_found" + + # Fall through: surface the concrete exception class. + return _to_snake_case(cls_name) From 5bf0727bc6a4a61b870bbd0e290dcb6a9e0a7946 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:20:14 +0800 Subject: [PATCH 15/24] feat(EverCore): JSON log output via LOG_FORMAT=json, no caller changes Per Phase 1, T1.2 of the observability roadmap. The logger module now selects its output format from the LOG_FORMAT env var: - LOG_FORMAT=text (default) keeps the existing human-readable format used by local development and tail-based debugging. - LOG_FORMAT=json switches to one JSON object per line via the new JsonFormatter. Each record carries ts (ISO 8601 ms + Z), level, logger, msg, file, request_id, and any extra= mapping the caller attached. Tracebacks live in a structured exc_info field instead of being concatenated into msg. ContextFilter (formerly RequestIdFilter) now also forwards tenant_id, user_id, group_id and session_id from the app_info contextvar onto the LogRecord, so JSON output captures them automatically when they are populated by middleware. The change is fully backward compatible: all 247 existing call sites keep working unchanged in both formats. Structured fields can be adopted incrementally via logger.info(msg, extra={...}) without touching the rest. Implemented with zero new dependencies (stdlib logging + json). A future migration to structlog can layer on top without churning callers again. Smoke-tested locally: with LOG_FORMAT=json, /readyz emits per-probe JSON lines that share the same request_id across the Redis / MongoDB / Elasticsearch / Milvus checks, which is exactly the cross-call correlation the roadmap calls out as Phase 1's headline win. --- .../EverCore/src/core/observation/logger.py | 166 +++++++++++++++--- 1 file changed, 140 insertions(+), 26 deletions(-) diff --git a/methods/EverCore/src/core/observation/logger.py b/methods/EverCore/src/core/observation/logger.py index f86a0865..9cb9f0dc 100644 --- a/methods/EverCore/src/core/observation/logger.py +++ b/methods/EverCore/src/core/observation/logger.py @@ -1,6 +1,7 @@ +import json import logging import traceback -from typing import Optional +from typing import Any, Dict, Optional from enum import Enum from functools import lru_cache import sys @@ -9,20 +10,117 @@ from core.context.context import get_current_app_info -class RequestIdFilter(logging.Filter): - """Injects request_id from app_info_context into every LogRecord. +# Keys we lift from app_info onto every LogRecord so they are available +# both as %(name)s format specifiers and as JSON output fields. Extra +# keys present in app_info are also forwarded (see ContextFilter). +_CORE_CONTEXT_KEYS = ("request_id", "tenant_id", "user_id", "group_id", "session_id") - Reads from the ContextVar-based app_info dict. Falls back to "-" - when no request context is available (startup, background tasks - without inherited context, etc.). + +class ContextFilter(logging.Filter): + """Promote contextvars-based ``app_info`` fields onto every LogRecord. + + Reads from the ``app_info_context`` ContextVar populated by middleware. + ``request_id`` is always set (defaults to ``"-"``) because the text + formatter references it. Other well-known context keys are forwarded + when present so JSON output captures them automatically. """ def filter(self, record: logging.LogRecord) -> bool: - app_info = get_current_app_info() - record.request_id = app_info.get("request_id", "-") if app_info else "-" + app_info = get_current_app_info() or {} + # request_id has a default because %(request_id)s is in the text + # formatter and would raise KeyError otherwise. + record.request_id = app_info.get("request_id", "-") + for key in _CORE_CONTEXT_KEYS[1:]: + if key in app_info: + setattr(record, key, app_info[key]) return True +# LogRecord attributes that are part of the stdlib API and should not be +# echoed back as user-supplied "extra" fields in JSON output. +_RESERVED_LOG_ATTRS = frozenset( + { + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "message", + "asctime", + "taskName", + } +) + + +class JsonFormatter(logging.Formatter): + """Render a LogRecord as a single line of JSON. + + Emits the standard fields plus anything that ``ContextFilter`` or a + caller's ``extra=...`` mapping attached to the record. Tracebacks are + included under the ``exc_info`` key, not concatenated into the + message, so downstream log shippers can keep them structured. + """ + + def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) -> str: + """ISO 8601 timestamp with millisecond precision and UTC suffix. + + Overrides the default which uses ``time.strftime`` (no ``%f`` + support) and so cannot express sub-second precision. + """ + from datetime import datetime, timezone + + return ( + datetime.fromtimestamp(record.created, tz=timezone.utc) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z") + ) + + def format(self, record: logging.LogRecord) -> str: + payload: Dict[str, Any] = { + "ts": self.formatTime(record), + "level": record.levelname, + "logger": record.name, + "msg": record.getMessage(), + "file": f"{record.filename}:{record.lineno}", + } + + # request_id is always populated by ContextFilter. + payload["request_id"] = getattr(record, "request_id", "-") + + # Forward any contextual or caller-supplied extras (anything not in + # the stdlib attribute set). + for key, value in record.__dict__.items(): + if key in _RESERVED_LOG_ATTRS or key == "request_id": + continue + if key.startswith("_"): + continue + payload[key] = value + + if record.exc_info: + payload["exc_info"] = self.formatException(record.exc_info) + if record.stack_info: + payload["stack_info"] = self.formatStack(record.stack_info) + + # default=str keeps datetimes / ObjectIds / Exception objects from + # crashing the formatter on unexpected types. + return json.dumps(payload, ensure_ascii=False, default=str) + + class LogLevel(Enum): """Log level enumeration""" @@ -52,30 +150,46 @@ def __init__(self): self._initialized = True def _setup_root_logging(self): - """Set up logging configuration""" - # Get log level from environment variable, default to INFO - log_level = os.getenv('LOG_LEVEL', 'INFO').upper() + """Set up logging configuration. + + Output format is selected by the ``LOG_FORMAT`` environment + variable: - # Configure root logger - # force=True ensures handlers are replaced even if root logger - # was already configured (e.g. by a library imported before us) + * ``text`` (default) — the human-readable format used historically. + Best for local development and existing tail-based debugging. + * ``json`` — one JSON object per line via :class:`JsonFormatter`. + Use in production so log shippers can index ``request_id``, + ``tenant_id``, exception class, etc. as structured fields. + + Existing call sites (``logger.info("text %s", value)``) work + unchanged in both formats. Switching to JSON does not require + touching any of the 247 logger call sites; structured fields can + be added incrementally via ``logger.info(msg, extra={...})``. + """ + log_level = os.getenv('LOG_LEVEL', 'INFO').upper() + log_format = os.getenv('LOG_FORMAT', 'text').lower() + + handler = logging.StreamHandler(sys.stdout) + handler.addFilter(ContextFilter()) + + if log_format == 'json': + handler.setFormatter(JsonFormatter()) + else: + handler.setFormatter( + logging.Formatter( + fmt='%(asctime)s - %(name)s - %(levelname)s - ' + '%(filename)s:%(lineno)d - [%(request_id)s] - %(message)s' + ) + ) + + # force=True drops any handlers a third-party library may have + # attached to the root logger before our setup runs. logging.basicConfig( level=getattr(logging, log_level), - format='%(asctime)s - %(name)s - %(levelname)s - %(filename)s:%(lineno)d - [%(request_id)s] - %(message)s', - handlers=[ - logging.StreamHandler(sys.stdout) - # Can add file handler - # logging.FileHandler('app.log', encoding='utf-8') - ], + handlers=[handler], force=True, ) - # Register RequestIdFilter on all root handlers so child loggers - # inherit it via propagation - request_id_filter = RequestIdFilter() - for handler in logging.root.handlers: - handler.addFilter(request_id_filter) - def _setup_logging(self): """Set up logging configuration""" self._setup_root_logging() From b4b2b817db557a12afbf36f9f42d22f26dac8e8d Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:48:08 +0800 Subject: [PATCH 16/24] docs(EverCore): add SLO definitions and Grafana dashboard skeleton Closes Phase 1 T1.5 of the code-quality roadmap. - docs/dev_docs/slo_definitions.md: four load-bearing SLOs (memorize success rate, retrieve p95 latency, LLM error budget, dependency availability) wired to existing metric names. Includes PromQL, severity guidance, and an error-budget policy. - docs/grafana/evercore_slo_overview.json: importable dashboard with one stat per SLO plus 5-min burn-rate timeseries panels. --- .../EverCore/docs/dev_docs/slo_definitions.md | 263 ++++++++++++++ methods/EverCore/docs/grafana/README.md | 19 + .../docs/grafana/evercore_slo_overview.json | 329 ++++++++++++++++++ 3 files changed, 611 insertions(+) create mode 100644 methods/EverCore/docs/dev_docs/slo_definitions.md create mode 100644 methods/EverCore/docs/grafana/README.md create mode 100644 methods/EverCore/docs/grafana/evercore_slo_overview.json diff --git a/methods/EverCore/docs/dev_docs/slo_definitions.md b/methods/EverCore/docs/dev_docs/slo_definitions.md new file mode 100644 index 00000000..a1f63e56 --- /dev/null +++ b/methods/EverCore/docs/dev_docs/slo_definitions.md @@ -0,0 +1,263 @@ +# SLO / SLI Definitions + +**Project**: EverCore +**Owner**: Platform / Observability +**Last reviewed**: 2026-05-26 +**Companion documents**: +- [`code_quality_roadmap.md`](./code_quality_roadmap.md) — Phase 1 T1.5 +- [`metrics_library_design.md`](./metrics_library_design.md) — metric naming and library + +--- + +## 1. Purpose + +This document is the **single source of truth** for what "healthy" means in +EverCore. It exists so that: + +- On-call engineers know which alerts are load-bearing and which are noise. +- Refactoring work (notably Phase 3 of the code-quality roadmap) has a + baseline to measure against — "did we make it better, worse, or neither?" +- Capacity, performance, and reliability work share the same yardsticks. + +If an SLI is not listed here, treat it as informational. Only the SLIs in +this document are tied to error budgets, paging, and roadmap exit criteria. + +--- + +## 2. Definitions + +- **SLI** (Service Level Indicator): a quantitative measure of one aspect of + service quality — e.g. "fraction of memorize requests that succeed". +- **SLO** (Service Level Objective): the target value or range for an SLI + over a stated window — e.g. "≥ 99.9% over a rolling 30 days". +- **Error budget**: `1 - SLO`. The amount of failure that is acceptable + before the SLO is breached. Used to gate risky changes. +- **Window**: every SLO below uses a **rolling 30-day** window unless + otherwise stated. Short bursts of failure are tolerated; sustained + failure is not. + +--- + +## 3. The SLO Set + +EverCore exposes three user-facing capabilities that warrant first-class +SLOs, plus one platform dependency SLO. Everything else is informational. + +### 3.1 SLO-1: `memorize_success_rate` + +**Statement**: At least **99.9%** of memorize requests succeed in a +rolling 30-day window. + +| Field | Value | +|---|---| +| SLI source | `evermemos_agentic_memorize_requests_total` | +| Success definition | `status="success"` **or** `status="extracted"` **or** `status="accumulated"` | +| Failure definition | `status="error"` | +| SLO target | ≥ 99.9% | +| Error budget | 0.1% (≈ 43.2 min / 30 d) | +| Window | rolling 30 d | +| Severity | **page** at 50% budget burn / 6 h; **ticket** at 25% burn / 24 h | + +**PromQL**: + +```promql +# Success rate over 30d +( + sum(rate(evermemos_agentic_memorize_requests_total{status=~"success|extracted|accumulated"}[30d])) + / + sum(rate(evermemos_agentic_memorize_requests_total[30d])) +) +``` + +**Notes**: +- `accumulated` is success — it means the message was queued for later + extraction, not that anything failed. +- Per-tenant rollups are available via the `space_id` label; do not include + `space_id` in the SLO numerator/denominator (high cardinality, and a + single noisy tenant should not move the global SLO). + +### 3.2 SLO-2: `retrieve_p95_latency` + +**Statement**: The **p95 end-to-end latency** of memory retrieval stays +**below 500 ms** in a rolling 30-day window. + +| Field | Value | +|---|---| +| SLI source | `evermemos_agentic_retrieve_duration_seconds` (Histogram) | +| Latency definition | server-side wall-clock, from controller entry to response serialised | +| SLO target | p95 < 500 ms | +| Error budget | 5% of buckets above the threshold (informally; latency budgets are time-burn based) | +| Window | rolling 30 d | +| Severity | **page** when p95 > 1 s for 10 min; **ticket** when p95 > 500 ms for 1 h | + +**PromQL**: + +```promql +# p95 over 30d, all memory types and retrieval methods +histogram_quantile( + 0.95, + sum by (le) (rate(evermemos_agentic_retrieve_duration_seconds_bucket[30d])) +) +``` + +**Notes**: +- Slice by `retrieve_method` (`vector`, `hybrid`, `rrf`, `agentic`, …) for + diagnosis. `agentic` retrieval is expected to be slower; an `agentic`-only + regression is not necessarily a global p95 regression. +- Stage-level latency lives in + `evermemos_agentic_retrieve_stage_duration_seconds`; use it during + incidents, not for the SLO itself. + +### 3.3 SLO-3: `llm_call_error_budget` + +**Statement**: At most **1%** of LLM API calls fail in a rolling 30-day +window. Counts retries collapsed — i.e. one call = one outcome after the +provider's last retry. + +| Field | Value | +|---|---| +| SLI source | `evermemos_memory_layer_llm_requests_total` | +| Success definition | `status="success"` | +| Failure definition | any of: `rate_limit`, `key_error`, `server_error`, `client_error`, `request_error` | +| SLO target | error rate < 1% | +| Error budget | 1% (i.e. up to 1 failure per 100 calls) | +| Window | rolling 30 d | +| Severity | **page** at `rate_limit` > 5% / 10 min (provider quota issue); **ticket** at total error rate > 1% / 1 h | + +**PromQL**: + +```promql +# Error rate over 30d, all models +( + sum(rate(evermemos_memory_layer_llm_requests_total{status!="success"}[30d])) + / + sum(rate(evermemos_memory_layer_llm_requests_total[30d])) +) +``` + +**Notes**: +- `rate_limit` and `key_error` indicate **provider-side** issues — quota, + billing, key rotation. They share the budget with code-side failures + because both visibly degrade user experience; the breakdown still helps + diagnose root cause. +- Slice by `model` to see whether a single model is dragging the budget. + +### 3.4 SLO-4 (platform): `dependency_availability` + +**Statement**: Each downstream dependency probed by `/readyz` is healthy +**≥ 99.95%** of the time over a rolling 30-day window. + +| Field | Value | +|---|---| +| SLI source | `evercore_dependency_healthy{name="..."}` (Gauge) | +| Dependencies | `redis`, `mongodb`, `elasticsearch`, `milvus`, `llm` | +| Healthy definition | gauge value `== 1` | +| SLO target | ≥ 99.95% per dependency | +| Window | rolling 30 d | +| Severity | **page** when any dependency is unhealthy for > 5 min; **ticket** when the gauge flaps (> 3 transitions / 10 min) | + +**PromQL**: + +```promql +# Per-dependency availability over 30d +avg_over_time(evercore_dependency_healthy[30d]) +``` + +**Notes**: +- This SLO measures **EverCore's view** of the dependency, not the + dependency itself. A network partition between EverCore and Redis + counts against this SLO even if Redis is fine. +- The `name` label values come from + `infra_layer/adapters/input/api/health/probes.py` — keep this list in + sync if probes are added or renamed. + +--- + +## 4. What is *not* an SLO + +To keep the SLO set load-bearing, the following are explicitly +**informational only** — observe them, do not page on them: + +- `boundary_detection_total`, `memcell_extracted_total`, + `memory_extracted_total` — quality signals, not availability signals. +- `retrieve_results_count` — product-quality metric; "zero results" is not + always an error. +- `memory_extraction_stage_duration_seconds`, + `retrieve_stage_duration_seconds` — diagnosis tools, not SLOs. Owning a + stage SLO would couple alerts to the current pipeline shape, which we + expect to evolve. +- Memory-pressure or queue-depth gauges — capacity signals; alert on + saturation, not on the gauge itself. + +When tempted to "promote" one of these to an SLO, ask: *can a user tell +when this is broken, and would they consider it broken?* If not, leave it +informational. + +--- + +## 5. Error Budget Policy + +Each user-facing SLO (SLO-1, SLO-2, SLO-3) has a 30-day error budget. + +- **Budget ≥ 50% remaining**: ship freely. Risky changes are acceptable. +- **Budget 25–50% remaining**: prefer reversible changes (feature flags, + canaries). Phase 3 refactors land behind flags only. +- **Budget < 25% remaining**: freeze non-critical changes. Focus on + reliability work until the burn rate flattens. +- **Budget exhausted**: stop feature work. The next merge must improve + the SLI driving the burn. + +The roadmap's **Phase 3** explicitly checks SLO health before each merge +(see [`code_quality_roadmap.md` §4.2](./code_quality_roadmap.md#42-deliverables)). + +--- + +## 6. Verification + +These checks should pass before declaring SLOs operational: + +```bash +# 1. Memorize success-rate metric is being recorded with the expected labels +curl -s localhost:1995/metrics | grep '^evermemos_agentic_memorize_requests_total' + +# 2. Retrieve latency histogram has the expected buckets +curl -s localhost:1995/metrics | grep '^evermemos_agentic_retrieve_duration_seconds_bucket' + +# 3. LLM request counter is using a non-"unknown" status (T1.3 acceptance) +curl -s localhost:1995/metrics | grep '^evermemos_memory_layer_llm_requests_total' + +# 4. Dependency gauges expose one series per downstream +curl -s localhost:1995/metrics | grep '^evercore_dependency_healthy' +``` + +A Grafana dashboard skeleton lives under `docs/grafana/`. Import it into +Grafana and point it at the Prometheus scraping `/metrics`. Per-tenant +breakdowns are intentionally omitted from the default dashboard to keep +cardinality controllable. + +--- + +## 7. Change Process + +- Adding or removing an SLO requires a PR that touches this document and + the relevant Grafana dashboard JSON in `docs/grafana/`. +- Tightening a target (lower latency, higher success rate) is a soft + change — open a PR, no separate review needed. +- Loosening a target (higher latency, lower success rate) requires + agreement from at least one other maintainer and a note explaining the + underlying constraint (e.g. provider rate-limit changes). +- All four SLOs are reviewed at the end of every roadmap phase and at + least once per quarter. + +--- + +## 8. References + +- [`metrics_library_design.md`](./metrics_library_design.md) — naming, + label conventions, registry singleton. +- [`exception_handling_analysis.md`](./exception_handling_analysis.md) §3 — + why `error_type='unknown'` was previously dominant, and how Phase 1 + T1.3 fixed it. +- `src/infra_layer/adapters/input/api/health/probes.py` — concrete list of + dependencies probed by `/readyz` and emitted as + `evercore_dependency_healthy`. diff --git a/methods/EverCore/docs/grafana/README.md b/methods/EverCore/docs/grafana/README.md new file mode 100644 index 00000000..b8b9831d --- /dev/null +++ b/methods/EverCore/docs/grafana/README.md @@ -0,0 +1,19 @@ +# Grafana Dashboards + +Dashboard JSON for the SLOs defined in +[`../dev_docs/slo_definitions.md`](../dev_docs/slo_definitions.md). + +## Files + +- `evercore_slo_overview.json` — one row per SLO (memorize success rate, + retrieve p95 latency, LLM error rate, dependency availability). + +## Import + +1. Open Grafana → Dashboards → Import. +2. Paste the JSON, or upload the file. +3. Pick the Prometheus data source that scrapes EverCore's `/metrics`. + +The default panels intentionally do not break down by `space_id` to keep +cardinality bounded; clone the dashboard and add per-tenant filters when +diagnosing a specific tenant. diff --git a/methods/EverCore/docs/grafana/evercore_slo_overview.json b/methods/EverCore/docs/grafana/evercore_slo_overview.json new file mode 100644 index 00000000..a273a970 --- /dev/null +++ b/methods/EverCore/docs/grafana/evercore_slo_overview.json @@ -0,0 +1,329 @@ +{ + "annotations": { + "list": [] + }, + "description": "EverCore SLO overview. See docs/dev_docs/slo_definitions.md for full definitions.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0.99, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 0.995}, + {"color": "green", "value": 0.999} + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0}, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(evermemos_agentic_memorize_requests_total{status=~\"success|extracted|accumulated\"}[30d])) / sum(rate(evermemos_agentic_memorize_requests_total[30d]))", + "legendFormat": "memorize success", + "refId": "A" + } + ], + "title": "SLO-1: memorize_success_rate (target ≥ 99.9%)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.5}, + {"color": "red", "value": 1.0} + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0}, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(evermemos_agentic_retrieve_duration_seconds_bucket[30d])))", + "legendFormat": "retrieve p95", + "refId": "A" + } + ], + "title": "SLO-2: retrieve_p95_latency (target < 500 ms)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "mappings": [], + "max": 0.05, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.005}, + {"color": "red", "value": 0.01} + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0}, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(evermemos_memory_layer_llm_requests_total{status!=\"success\"}[30d])) / sum(rate(evermemos_memory_layer_llm_requests_total[30d]))", + "legendFormat": "llm error rate", + "refId": "A" + } + ], + "title": "SLO-3: llm_call_error_budget (target < 1%)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "mappings": [], + "max": 1, + "min": 0.99, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 0.999}, + {"color": "green", "value": 0.9995} + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0}, + "id": 4, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg_over_time(evercore_dependency_healthy[30d])", + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "title": "SLO-4: dependency_availability (target ≥ 99.95% each)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": true + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, + "id": 5, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(evermemos_agentic_memorize_requests_total{status=~\"success|extracted|accumulated\"}[5m])) / sum(rate(evermemos_agentic_memorize_requests_total[5m]))", + "legendFormat": "memorize 5m success", + "refId": "A" + } + ], + "title": "Memorize success rate (5m rolling, for burn-rate context)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": true + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, + "id": 6, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, retrieve_method) (rate(evermemos_agentic_retrieve_duration_seconds_bucket[5m])))", + "legendFormat": "p95 {{retrieve_method}}", + "refId": "A" + } + ], + "title": "Retrieve p95 by method (5m rolling)", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "tags": ["evercore", "slo"], + "templating": { + "list": [ + { + "current": {"selected": false, "text": "Prometheus", "value": "Prometheus"}, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": {"from": "now-24h", "to": "now"}, + "timepicker": {}, + "timezone": "", + "title": "EverCore — SLO Overview", + "uid": "evercore-slo-overview", + "version": 1 +} From 1e1e015e8144dec89398da8436bba9d070790dd2 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:48:39 +0800 Subject: [PATCH 17/24] feat(EverCore): OpenTelemetry opt-in tracing skeleton Closes Phase 1 T1.4 of the code-quality roadmap. - core/observation/tracing/otel.py: soft-imports the OTel SDK; init_tracing is a no-op unless OTEL_EXPORTER_OTLP_ENDPOINT is set AND the deps are installed. Installs FastAPI/httpx/redis/pymongo instrumentations when available. Default sampler is ParentBased(TraceIdRatioBased(0.1)). - trace_logger decorator now opens a real span when tracing is active; unchanged behavior when inactive (zero perf cost). - base_app wires init_tracing(app) at the end of create_base_app. - pyproject adds an optional `otel` dependency group. Activate with `uv sync --group otel`; default sync does NOT pull these in. --- methods/EverCore/pyproject.toml | 14 + methods/EverCore/src/base_app.py | 5 + .../core/observation/tracing/decorators.py | 127 +++++---- .../src/core/observation/tracing/otel.py | 221 +++++++++++++++ methods/EverCore/uv.lock | 260 ++++++++++++++++++ 5 files changed, 579 insertions(+), 48 deletions(-) create mode 100644 methods/EverCore/src/core/observation/tracing/otel.py diff --git a/methods/EverCore/pyproject.toml b/methods/EverCore/pyproject.toml index 283545c7..cd7b9c23 100644 --- a/methods/EverCore/pyproject.toml +++ b/methods/EverCore/pyproject.toml @@ -219,6 +219,20 @@ cpu = [ gpu = [ ] +# OpenTelemetry — optional, opt-in. +# Activate at runtime via OTEL_EXPORTER_OTLP_ENDPOINT and install with: +# uv sync --group otel +# Default sync does NOT pull these in; the app stays no-op if not installed. +otel = [ + "opentelemetry-api>=1.27.0", + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp>=1.27.0", + "opentelemetry-instrumentation-fastapi>=0.48b0", + "opentelemetry-instrumentation-httpx>=0.48b0", + "opentelemetry-instrumentation-redis>=0.48b0", + "opentelemetry-instrumentation-pymongo>=0.48b0", +] + # Full development environment - includes dev tools dev-full = [ {include-group = "dev"}, diff --git a/methods/EverCore/src/base_app.py b/methods/EverCore/src/base_app.py index edd15d3c..27a4f01d 100644 --- a/methods/EverCore/src/base_app.py +++ b/methods/EverCore/src/base_app.py @@ -10,6 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware from core.observation.logger import get_logger +from core.observation.tracing.otel import init_tracing from core.middleware.global_exception_handler import global_exception_handler from core.middleware.profile_middleware import ProfileMiddleware from core.di.utils import get_bean_by_type @@ -103,6 +104,10 @@ def create_base_app( # Mount lifespan management methods to app instance _mount_lifespan_methods(app) + # Initialise OpenTelemetry. No-op unless OTEL_EXPORTER_OTLP_ENDPOINT is + # set AND the optional `otel` dependency group is installed. + init_tracing(app) + return app diff --git a/methods/EverCore/src/core/observation/tracing/decorators.py b/methods/EverCore/src/core/observation/tracing/decorators.py index 0c2e12b4..afbb9d43 100644 --- a/methods/EverCore/src/core/observation/tracing/decorators.py +++ b/methods/EverCore/src/core/observation/tracing/decorators.py @@ -6,12 +6,31 @@ from functools import wraps from typing import Callable, Optional +from contextlib import contextmanager import logging import time +from core.observation.tracing.otel import get_tracer + logger = logging.getLogger(__name__) +@contextmanager +def _otel_span(operation: str, module: str): + """Open an OTel span if tracing is active; otherwise a no-op. + + Kept here (not in otel.py) so that the import surface in decorators + stays minimal and the soft-import logic in otel.py remains the single + source of truth for OTel availability. + """ + tracer = get_tracer(module) + if tracer is None: + yield None + return + with tracer.start_as_current_span(operation) as span: + yield span + + def trace_logger( operation_name: Optional[str] = None, include_args: bool = False, @@ -49,30 +68,36 @@ async def async_wrapper(*args, **kwargs): _log_message(logger, log_level, log_message) - try: - # Execute original function - result = await func(*args, **kwargs) - - # Log success completion message - end_time = time.time() - duration = round((end_time - start_time) * 1000, 2) # milliseconds - - log_message = f"\n\t[trace] {operation} - Processing completed (duration: {duration}ms)" - if include_result and result is not None: - result_str = _format_result(result) - log_message += f" | Result: {result_str}" - - _log_message(logger, log_level, log_message) - return result - - except Exception as e: - # Log exception message - end_time = time.time() - duration = round((end_time - start_time) * 1000, 2) - - log_message = f"\n\t[trace] {operation} - Processing failed (duration: {duration}ms) | Error: {str(e)}" - _log_message(logger, "error", log_message) - raise + with _otel_span(operation, func.__module__) as span: + try: + # Execute original function + result = await func(*args, **kwargs) + + # Log success completion message + end_time = time.time() + duration = round((end_time - start_time) * 1000, 2) # milliseconds + + log_message = f"\n\t[trace] {operation} - Processing completed (duration: {duration}ms)" + if include_result and result is not None: + result_str = _format_result(result) + log_message += f" | Result: {result_str}" + + _log_message(logger, log_level, log_message) + if span is not None: + span.set_attribute("duration_ms", duration) + return result + + except Exception as e: + # Log exception message + end_time = time.time() + duration = round((end_time - start_time) * 1000, 2) + + log_message = f"\n\t[trace] {operation} - Processing failed (duration: {duration}ms) | Error: {str(e)}" + _log_message(logger, "error", log_message) + if span is not None: + span.record_exception(e) + span.set_attribute("duration_ms", duration) + raise @wraps(func) def sync_wrapper(*args, **kwargs): @@ -94,30 +119,36 @@ def sync_wrapper(*args, **kwargs): _log_message(logger, log_level, log_message) - try: - # Execute original function - result = func(*args, **kwargs) - - # Log success completion message - end_time = time.time() - duration = round((end_time - start_time) * 1000, 2) - - log_message = f"\n\t[trace] {operation} - Processing completed (duration: {duration}ms)" - if include_result and result is not None: - result_str = _format_result(result) - log_message += f" | Result: {result_str}" - - _log_message(logger, log_level, log_message) - return result - - except Exception as e: - # Log exception message - end_time = time.time() - duration = round((end_time - start_time) * 1000, 2) - - log_message = f"\n\t[trace] {operation} - Processing failed (duration: {duration}ms) | Error: {str(e)}" - _log_message(logger, "error", log_message) - raise + with _otel_span(operation, func.__module__) as span: + try: + # Execute original function + result = func(*args, **kwargs) + + # Log success completion message + end_time = time.time() + duration = round((end_time - start_time) * 1000, 2) + + log_message = f"\n\t[trace] {operation} - Processing completed (duration: {duration}ms)" + if include_result and result is not None: + result_str = _format_result(result) + log_message += f" | Result: {result_str}" + + _log_message(logger, log_level, log_message) + if span is not None: + span.set_attribute("duration_ms", duration) + return result + + except Exception as e: + # Log exception message + end_time = time.time() + duration = round((end_time - start_time) * 1000, 2) + + log_message = f"\n\t[trace] {operation} - Processing failed (duration: {duration}ms) | Error: {str(e)}" + _log_message(logger, "error", log_message) + if span is not None: + span.record_exception(e) + span.set_attribute("duration_ms", duration) + raise # Return corresponding wrapper based on whether the function is a coroutine import asyncio diff --git a/methods/EverCore/src/core/observation/tracing/otel.py b/methods/EverCore/src/core/observation/tracing/otel.py new file mode 100644 index 00000000..9aaaac4d --- /dev/null +++ b/methods/EverCore/src/core/observation/tracing/otel.py @@ -0,0 +1,221 @@ +"""OpenTelemetry initialisation. + +This module is **opt-in**. EverCore ships without OpenTelemetry runtime +dependencies; install the ``otel`` dependency group to enable it: + + uv sync --group otel + +At import time we soft-import the OTel SDK. If the SDK is missing, every +public function in this module degrades to a no-op so application startup +is not affected. + +Activation is further gated on the ``OTEL_EXPORTER_OTLP_ENDPOINT`` +environment variable. When that variable is unset, ``init_tracing`` is a +no-op even if the SDK is installed. This means a developer can install +the ``otel`` group without immediately exporting spans — useful for tests. + +Configuration (read from environment): + +- ``OTEL_EXPORTER_OTLP_ENDPOINT`` — collector endpoint (e.g. + ``http://jaeger:4317``). Required to enable tracing. +- ``OTEL_SERVICE_NAME`` — service name reported in spans. Defaults to + ``evercore``. +- ``OTEL_TRACES_SAMPLER_ARG`` — sampling ratio, 0.0–1.0. Defaults to + 0.1 (10%) to keep collector load bounded. +- ``OTEL_EXPORTER_OTLP_PROTOCOL`` — ``grpc`` (default) or ``http/protobuf``. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Soft-import OpenTelemetry. Anything imported under ``_otel_*`` is either a +# real OTel object or ``None``. Callers check the flags below before using. +# --------------------------------------------------------------------------- + +try: + from opentelemetry import trace as _otel_trace + from opentelemetry.sdk.resources import Resource as _OtelResource + from opentelemetry.sdk.trace import TracerProvider as _OtelTracerProvider + from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor as _OtelBatchSpanProcessor, + ) + from opentelemetry.sdk.trace.sampling import ( + ParentBased as _OtelParentBased, + TraceIdRatioBased as _OtelTraceIdRatioBased, + ) + + _OTEL_AVAILABLE = True +except ImportError: + _otel_trace = None # type: ignore[assignment] + _OtelResource = None # type: ignore[assignment] + _OtelTracerProvider = None # type: ignore[assignment] + _OtelBatchSpanProcessor = None # type: ignore[assignment] + _OtelParentBased = None # type: ignore[assignment] + _OtelTraceIdRatioBased = None # type: ignore[assignment] + _OTEL_AVAILABLE = False + + +# Process-wide flag set after ``init_tracing`` succeeds. ``trace_logger`` +# checks this to decide whether to open spans. +_TRACING_ACTIVE: bool = False + + +def is_tracing_active() -> bool: + """Return True iff OTel was initialised and is exporting spans.""" + return _TRACING_ACTIVE + + +def init_tracing(app: Optional[Any] = None) -> bool: + """Initialise OpenTelemetry tracing if the environment requests it. + + Safe to call multiple times — subsequent calls are no-ops. + + Args: + app: Optional FastAPI application. When passed and the + ``opentelemetry-instrumentation-fastapi`` package is + available, the app is instrumented for incoming-request + spans. + + Returns: + True if tracing was activated, False otherwise. Callers can + use this to log or surface the state at startup. + """ + global _TRACING_ACTIVE + + if _TRACING_ACTIVE: + return True + + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() + if not endpoint: + logger.debug( + "OpenTelemetry disabled: OTEL_EXPORTER_OTLP_ENDPOINT not set" + ) + return False + + if not _OTEL_AVAILABLE: + logger.warning( + "OTEL_EXPORTER_OTLP_ENDPOINT is set but opentelemetry SDK is " + "not installed. Install the 'otel' dependency group " + "(`uv sync --group otel`) to enable tracing." + ) + return False + + service_name = os.environ.get("OTEL_SERVICE_NAME", "evercore") + sample_ratio = _read_float_env("OTEL_TRACES_SAMPLER_ARG", default=0.1) + + resource = _OtelResource.create({"service.name": service_name}) + sampler = _OtelParentBased(root=_OtelTraceIdRatioBased(sample_ratio)) + provider = _OtelTracerProvider(resource=resource, sampler=sampler) + + exporter = _build_otlp_exporter(endpoint) + if exporter is None: + return False + + provider.add_span_processor(_OtelBatchSpanProcessor(exporter)) + _otel_trace.set_tracer_provider(provider) + + _install_optional_instrumentations(app) + + _TRACING_ACTIVE = True + logger.info( + "OpenTelemetry tracing active: endpoint=%s service=%s sample_ratio=%s", + endpoint, + service_name, + sample_ratio, + ) + return True + + +def get_tracer(name: str): + """Return an OTel tracer, or None if tracing is inactive. + + The trace_logger decorator uses this to decide whether to open a span. + """ + if not _TRACING_ACTIVE or _otel_trace is None: + return None + return _otel_trace.get_tracer(name) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _read_float_env(name: str, default: float) -> float: + raw = os.environ.get(name) + if raw is None: + return default + try: + value = float(raw) + except ValueError: + logger.warning("Invalid float for %s=%r; using default %s", name, raw, default) + return default + if not 0.0 <= value <= 1.0: + logger.warning("%s=%s outside [0,1]; clamping", name, value) + return max(0.0, min(1.0, value)) + return value + + +def _build_otlp_exporter(endpoint: str): + """Build an OTLP exporter, preferring gRPC and falling back to HTTP.""" + protocol = os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() + + if protocol in ("grpc", ""): + try: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + + return OTLPSpanExporter(endpoint=endpoint, insecure=True) + except ImportError: + logger.warning( + "OTLP gRPC exporter not installed; trying HTTP exporter" + ) + + try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + + return OTLPSpanExporter(endpoint=endpoint) + except ImportError: + logger.error( + "No OTLP exporter available; install opentelemetry-exporter-otlp" + ) + return None + + +def _install_optional_instrumentations(app: Optional[Any]) -> None: + """Install any auto-instrumentations whose packages are present. + + Each instrumentation is wrapped in its own try/except so missing + packages do not prevent the rest from loading. + """ + if app is not None: + try: + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + FastAPIInstrumentor.instrument_app(app) + except ImportError: + logger.debug("FastAPI instrumentation not installed; skipping") + + for module_path, instrumentor_attr in ( + ("opentelemetry.instrumentation.httpx", "HTTPXClientInstrumentor"), + ("opentelemetry.instrumentation.redis", "RedisInstrumentor"), + ("opentelemetry.instrumentation.pymongo", "PymongoInstrumentor"), + ): + try: + module = __import__(module_path, fromlist=[instrumentor_attr]) + getattr(module, instrumentor_attr)().instrument() + except ImportError: + logger.debug("%s not installed; skipping", module_path) + except Exception as exc: # pragma: no cover — instrumentor-side failure # noqa: BLE001 + logger.warning( + "Failed to enable %s instrumentation: %s", module_path, exc + ) diff --git a/methods/EverCore/uv.lock b/methods/EverCore/uv.lock index a82c451e..9d1205c3 100644 --- a/methods/EverCore/uv.lock +++ b/methods/EverCore/uv.lock @@ -158,6 +158,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/b3/a24a183c628da633b7cafd1759b14aaf47958de82ba6bcae9f1c2898781d/arq-0.26.3-py3-none-any.whl", hash = "sha256:9f4b78149a58c9dc4b88454861a254b7c4e7a159f2c973c89b548288b77e9005", size = 25968, upload-time = "2025-01-06T22:44:45.771Z" }, ] +[[package]] +name = "asgiref" +version = "3.11.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/40/f03da1264ae8f7cfdbf9146542e5e7e8100a4c66ab48e791df9a03d3f6c0/asgiref-3.11.1.tar.gz", hash = "sha256:5f184dc43b7e763efe848065441eac62229c9f7b0475f41f80e207a114eda4ce", size = 38550, upload-time = "2026-02-03T13:30:14.33Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/0a/a72d10ed65068e115044937873362e6e32fab1b7dce0046aeb224682c989/asgiref-3.11.1-py3-none-any.whl", hash = "sha256:e8667a091e69529631969fd45dc268fa79b99c92c5fcdda727757e52146ec133", size = 24345, upload-time = "2026-02-03T13:30:13.039Z" }, +] + [[package]] name = "asttokens" version = "3.0.1" @@ -1569,6 +1578,15 @@ evaluation-full = [ { name = "rich" }, { name = "zep-cloud" }, ] +otel = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-instrumentation-fastapi" }, + { name = "opentelemetry-instrumentation-httpx" }, + { name = "opentelemetry-instrumentation-pymongo" }, + { name = "opentelemetry-instrumentation-redis" }, + { name = "opentelemetry-sdk" }, +] [package.metadata] requires-dist = [ @@ -1675,6 +1693,15 @@ evaluation-full = [ { name = "zep-cloud", specifier = ">=2.0.0" }, ] gpu = [] +otel = [ + { name = "opentelemetry-api", specifier = ">=1.27.0" }, + { name = "opentelemetry-exporter-otlp", specifier = ">=1.27.0" }, + { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.48b0" }, + { name = "opentelemetry-instrumentation-httpx", specifier = ">=0.48b0" }, + { name = "opentelemetry-instrumentation-pymongo", specifier = ">=0.48b0" }, + { name = "opentelemetry-instrumentation-redis", specifier = ">=0.48b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, +] prod = [] [[package]] @@ -1798,6 +1825,219 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/59/fd/ae2da789cd923dd033c99b8d544071a827c92046b150db01cfa5cea5b3fd/openai-2.9.0-py3-none-any.whl", hash = "sha256:0d168a490fbb45630ad508a6f3022013c155a68fd708069b6a1a01a5e8f0ffad", size = 1030836, upload-time = "2025-12-04T18:15:07.063Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/1c/125e1c936c0873796771b7f04f6c93b9f1bf5d424cea90fda94a99f61da8/opentelemetry_api-1.42.1.tar.gz", hash = "sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716", size = 72296, upload-time = "2026-05-21T16:32:49.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/ca/9520cc1f3dfbbd03ac5903bbf55833e257bc64b1cf30fa8b0d6df374d821/opentelemetry_api-1.42.1-py3-none-any.whl", hash = "sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714", size = 61311, upload-time = "2026-05-21T16:32:28.822Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/94/8637919a5d01f81dacf510234bc0110b944f4687a6e96b0a02adf2f6bdce/opentelemetry_exporter_otlp-1.42.1.tar.gz", hash = "sha256:2d9ebaed714377a67d224d46795ddcc11d2c877fa5de35fda70b6f3b010729a9", size = 6086, upload-time = "2026-05-21T16:32:51.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/4d/c26080295a36fd22e201fefd7cb9c22cd203189b1af8cd73b158382b7ad8/opentelemetry_exporter_otlp-1.42.1-py3-none-any.whl", hash = "sha256:aedd54545bb0587cd45210abdc8be545af9c01413f3307786e276df1e3c83bee", size = 6733, upload-time = "2026-05-21T16:32:31.261Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/9c/216acfeaedadf2e1937f4373929b20f73197c5c4a2546d4f584b7fa63813/opentelemetry_exporter_otlp_proto_common-1.42.1.tar.gz", hash = "sha256:04f1f01fb597c4249dfcd7f8b861c902c2102369d376d9d346ff38de4469a2ee", size = 21433, upload-time = "2026-05-21T16:32:55.526Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/43/2375e7612e1121a4518c17603b6e0b03ad94f565aafad53f464dc5be2bf6/opentelemetry_exporter_otlp_proto_common-1.42.1-py3-none-any.whl", hash = "sha256:f48d395ab815b444da118868977e9798ea354c25737d5cf39578ae894011c140", size = 17327, upload-time = "2026-05-21T16:32:33.387Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/87/ca7fc790dfdbcf4f9e9aab14a39ef1b7508ead13707e283de0b3131478d2/opentelemetry_exporter_otlp_proto_grpc-1.42.1.tar.gz", hash = "sha256:975c4461f167dd8ed8857d68d3b6b25f3d272eab896f6a9470d0f5b90e2faf15", size = 27140, upload-time = "2026-05-21T16:32:56.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/2b/28ba5b128f47fe8c3bab541000d6feb4b5a9bd26623ca013406f01c0fb60/opentelemetry_exporter_otlp_proto_grpc-1.42.1-py3-none-any.whl", hash = "sha256:0ae1177e2038b18a929b3098215243631ef91136cba26b7e2b12790ceb7e87cc", size = 19617, upload-time = "2026-05-21T16:32:34.278Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/32/826bfa1d80ecea24f47808de03cd4a0d13c17ecc07712f45123f0f61e4ac/opentelemetry_exporter_otlp_proto_http-1.42.1.tar.gz", hash = "sha256:bf142a21035d7571ac3a09cb2e5639f49886f243972883cfe777ed3bf02b734d", size = 25406, upload-time = "2026-05-21T16:32:56.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/96/82cb223a1502f0787d4bbff12907f5f8d870a50731febcd5818d93ef9555/opentelemetry_exporter_otlp_proto_http-1.42.1-py3-none-any.whl", hash = "sha256:00a16da1b312a1d6c7233d600d557c91df71125af73020f3b9a7765bd699d59d", size = 21793, upload-time = "2026-05-21T16:32:35.277Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/6d/4de72d97ff54db1ed270c7a59c9b904b917c0ac7af429c086c388b824ddb/opentelemetry_instrumentation-0.63b1.tar.gz", hash = "sha256:32368d6ae52c8de20aa790a6ad86b10a76f09956092337ae37d675773990e541", size = 41081, upload-time = "2026-05-21T16:36:14.206Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/a1/9314e621c143e4d82a5bf7a43c2ff7a745d31023506336857607c8c543cc/opentelemetry_instrumentation-0.63b1-py3-none-any.whl", hash = "sha256:f1986716d52cc316ea5f60189098726a9071d8ecc0eee96c9ed110be08bade9c", size = 35577, upload-time = "2026-05-21T16:34:56.818Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-asgi" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asgiref" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a0/b5/7ea3a9fd1b80e89786c14250bfaecf32a753c3fd08232690f4da8dc16e29/opentelemetry_instrumentation_asgi-0.63b1.tar.gz", hash = "sha256:267b422416d768f3c7f4054883b41d9c3a7c943d86d20032b738c99a3dbb5862", size = 26151, upload-time = "2026-05-21T16:36:18.368Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/7e/83986f27b421de04fab1e1a84e892621dac42e6432a9c66779505f4d1381/opentelemetry_instrumentation_asgi-0.63b1-py3-none-any.whl", hash = "sha256:1a22453dfa965f14799b10a674b8acbcb897a8a75c79136060af54214cc7886e", size = 15906, upload-time = "2026-05-21T16:35:04.162Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-fastapi" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-instrumentation-asgi" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/d6/0c128fac2e34b7d526a8d3c6edc45b875a97f8a987861b00511151b6337d/opentelemetry_instrumentation_fastapi-0.63b1.tar.gz", hash = "sha256:cc42dff56c96d0a2921510c4abab2a4c2e27fe64b26dc1254727fb550df100ba", size = 25387, upload-time = "2026-05-21T16:36:32.071Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/3d/2eae63f13f36d7a8ab5bf03d06ecaf169c2069b524547f24947be6d92094/opentelemetry_instrumentation_fastapi-0.63b1-py3-none-any.whl", hash = "sha256:52ee2cde9a2ac094bdd45d79f85860e03a972928a2553006071fe61d94cf7281", size = 12795, upload-time = "2026-05-21T16:35:28.68Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-httpx" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "opentelemetry-util-http" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/27/c2b4335bca030e893acbe5ff2b4f434868773bf94508be7e6bf5af981b24/opentelemetry_instrumentation_httpx-0.63b1.tar.gz", hash = "sha256:f41ec82f25c3abcdada621052db3e5fd648e3b43d55eec4b9c0c5d3ecb7b4ff4", size = 23557, upload-time = "2026-05-21T16:36:34.583Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/b8/f536780996195c3b9f2354998554671e05a7a262df8c043f63fe9e5a6f0b/opentelemetry_instrumentation_httpx-0.63b1-py3-none-any.whl", hash = "sha256:14df6e99d81be9a8cd238f6639b6fa52404c4d3ce219058fcb5dc8c0f2211f86", size = 16336, upload-time = "2026-05-21T16:35:32.221Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-pymongo" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/b2/94c180359165abe62e829250bc6ac6b7daa2334b3c505bad64f1c64f18ab/opentelemetry_instrumentation_pymongo-0.63b1.tar.gz", hash = "sha256:8c0ae185b59dcb45c80bf90d4ffda5fcc6337dbba11de40306ffd69459e476fa", size = 10208, upload-time = "2026-05-21T16:36:41.961Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/b9/47b8d0f52d81d7661debebb11f4fdd1ab869d694089711c1ac8988456364/opentelemetry_instrumentation_pymongo-0.63b1-py3-none-any.whl", hash = "sha256:0d8dd55b2522eda4a7093da8b5f47fae9a3235fb2786bc14c161d5999a66320d", size = 10290, upload-time = "2026-05-21T16:35:44.634Z" }, +] + +[[package]] +name = "opentelemetry-instrumentation-redis" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-instrumentation" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/58/2a91453c70943d6af4b5f9f1c232d69e6093800f95349ff5f1f8a89cf6ba/opentelemetry_instrumentation_redis-0.63b1.tar.gz", hash = "sha256:28d235159df43cc2bc8779af5c602afad1e08603fff75ac8ca34dd1bf30a9cb9", size = 16711, upload-time = "2026-05-21T16:36:44.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/17/33c21901325f6bf96939f355db174627c148c83211d0412622d4066f560d/opentelemetry_instrumentation_redis-0.63b1-py3-none-any.whl", hash = "sha256:f0e51c4006f68e340abbf28a7995feff004de78649697cbdf3bac0072cacd082", size = 14539, upload-time = "2026-05-21T16:35:49.995Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/55/63eac3e1089b768ba014091fdd2ae8a9a440c821ef5e2b786909c94c8836/opentelemetry_proto-1.42.1.tar.gz", hash = "sha256:c6a51e6b4f05ae63565f3a113217f3d2bfaec68f78c02d7a6c85f9010d1cfca6", size = 45839, upload-time = "2026-05-21T16:33:03.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/9d/171c02c84a76940b7e601805b3bb536985aded9168fbcc9ba52f0a730fa2/opentelemetry_proto-1.42.1-py3-none-any.whl", hash = "sha256:dedb74cba2886c59c7789b227a7a670613025a07489040050aedff6e5c0fb43c", size = 71782, upload-time = "2026-05-21T16:32:44.867Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f7/b390bd9bfd703bf98a68fea1f27786c6872331fd617164a54b8a59bdc008/opentelemetry_sdk-1.42.1.tar.gz", hash = "sha256:8c834e8f8c9ba4171d4ec843d0cb8a67e4c7394d3f9e9297e582cbd9456ddbf7", size = 239262, upload-time = "2026-05-21T16:33:04.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/6b/4287766cfbde577ae2272e8884abac325aeaac0d64f41c61d5b8cc595105/opentelemetry_sdk-1.42.1-py3-none-any.whl", hash = "sha256:083cd4bbfaa5aa7b5a9e552430d9951219967cfb27aa61feb13a77aba1fc839d", size = 170907, upload-time = "2026-05-21T16:32:45.894Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/99/4d7dd6df64795951413ce6e815f8cf1eb191daf7196ae86574589643d5f3/opentelemetry_semantic_conventions-0.63b1.tar.gz", hash = "sha256:3daf963611334b365e98a57438183eb012d3bfb40b2d931a9af613476b8701a9", size = 148340, upload-time = "2026-05-21T16:33:05.455Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7a/7fe66f5f3682b1dd47d88cc4e11f1c6c0966b737de2d16671146e23c39a5/opentelemetry_semantic_conventions-0.63b1-py3-none-any.whl", hash = "sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682", size = 203713, upload-time = "2026-05-21T16:32:47.016Z" }, +] + +[[package]] +name = "opentelemetry-util-http" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/d8/7bf5e4cec0578ac3c28c18eb7b88f34279139cbc8c568d6aa02b9c5ae53e/opentelemetry_util_http-0.63b1.tar.gz", hash = "sha256:ba1268f00922ee522dba2ae38458060f99486e7385a8056985901ca9685adfff", size = 11102, upload-time = "2026-05-21T16:36:56.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/f1/34e047e8f6a3c67e5220acf1af7b9f62868c25d77791bca74457bd2180a6/opentelemetry_util_http-0.63b1-py3-none-any.whl", hash = "sha256:6284194028c59cd439f8acfe388145069a6127f11dc077e1344a2094adacc3f8", size = 8205, upload-time = "2026-05-21T16:36:09.736Z" }, +] + [[package]] name = "orjson" version = "3.11.4" @@ -3103,6 +3343,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] +[[package]] +name = "wrapt" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/9f/06263fcd8ad6c405f05a3905fd7a84dd3176eb5ad46e44bccc0cd16348bb/wrapt-2.2.1.tar.gz", hash = "sha256:6744f504375775d7609c82c8d3d94af1c9a6f05586984536905908ba905277b9", size = 127620, upload-time = "2026-05-22T14:49:43.056Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/0c/bfae7b9401583b6d05938cd16dedc43857d96da2f8a3d50d78cc515bf6ff/wrapt-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3ffad790d9d11d8ecf9f17c4bb671a5b4089e4d8b575c46c5129597f41f836b0", size = 81021, upload-time = "2026-05-22T14:48:00.313Z" }, + { url = "https://files.pythonhosted.org/packages/26/58/80f6a6599f933f4caecc1cb3ee88a04faf81e8b9bddbd6109c688dd63e0f/wrapt-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:628f5220c7a904d5fc78f7075c8d7871433eb6d035c94728a22fdf85f193d2a8", size = 81692, upload-time = "2026-05-22T14:48:01.49Z" }, + { url = "https://files.pythonhosted.org/packages/17/93/fb357cc7847c58a8ae790be718903afa81a28d23e642c843dc4129e8a0b2/wrapt-2.2.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:61acce4257a9883669703c525447c5b4c392edf0f987ae77ec32668440158f0e", size = 169364, upload-time = "2026-05-22T14:48:02.791Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0b/76b601ee309a8bd556af0eecb184394c20b3c49aa9c8e085aa1ffacc2568/wrapt-2.2.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727ab4244622cd6ad2390f322642090c877d2e83a608d2653a7643ae5368d926", size = 171079, upload-time = "2026-05-22T14:48:04.22Z" }, + { url = "https://files.pythonhosted.org/packages/cd/87/ee3f32d5658e3e26d3e0e457922b47a36dd3bfbdfee7f97bb3e802344a66/wrapt-2.2.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:03df9ebed4c73ab93fa8c07e3d41d818dfca1852b15731a3de59457b27814624", size = 160205, upload-time = "2026-05-22T14:48:05.553Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d0/ae2fd64277a67f5d7bffcf2d05eea1e476263fb2a072baf0b0129ab85984/wrapt-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0d9ff006f420b2ec8296aa56ade43ea7da3e997e85769f0aafc5e0661aacb710", size = 168922, upload-time = "2026-05-22T14:48:07.132Z" }, + { url = "https://files.pythonhosted.org/packages/b1/f3/2d541a060c5bbafb9400bca4917e4d78bfd1f239f404782c86831a8f6b29/wrapt-2.2.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:844c858fc3bb7eacc0ba8efa904935d16aac6a4470948ad1e7e55c9f5a2a665f", size = 158388, upload-time = "2026-05-22T14:48:08.629Z" }, + { url = "https://files.pythonhosted.org/packages/1d/68/8d92c8800c57e93cb116ae9e9d6cbafc34fade5ee9f9107b6f203fb4dc35/wrapt-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:87bacdaf225117a342a20d9c03438d701c02112f6e3f351ce9b7f32354f14797", size = 167682, upload-time = "2026-05-22T14:48:10.042Z" }, + { url = "https://files.pythonhosted.org/packages/30/72/83ea3790ea352439442349388e29ff07b76e0686265f9088bbb505d1608d/wrapt-2.2.1-cp312-cp312-win32.whl", hash = "sha256:2f8c90c8afde51969487be4e1343ae049b268854877d415c2510baf833775052", size = 77857, upload-time = "2026-05-22T14:48:11.782Z" }, + { url = "https://files.pythonhosted.org/packages/ef/cb/99450668dd3502d62a54a1c8aa56e44f34cb8c1261b381cfe2e7926c3b75/wrapt-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ce32763ac31ce94fe9aada947e479b1975012bff166da409b4b9e4e376cf7e5", size = 80825, upload-time = "2026-05-22T14:48:13.046Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3a/87512881be64e743f9ee4c66f4cbe8e884974bef2a5989af71f999653ac7/wrapt-2.2.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d1b4d0e0c2119587a31f5c029abd547e0c81d93b89d394566fe1588659eb579", size = 79087, upload-time = "2026-05-22T14:48:14.323Z" }, + { url = "https://files.pythonhosted.org/packages/53/46/29ac9daf11a86c22a8c38cd9236c62928ccae83f7ceb06bd3b0467cf9d05/wrapt-2.2.1-py3-none-any.whl", hash = "sha256:3aafea2975caef8ca49400640dde02cc7426e798f24870ed01f490bc3cffd32f", size = 61000, upload-time = "2026-05-22T14:49:41.593Z" }, +] + [[package]] name = "xxhash" version = "3.6.0" From 3b6584ec3cef57551fa305be67407b5e8bbf579c Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:49:05 +0800 Subject: [PATCH 18/24] =?UTF-8?q?feat(EverCore):=20Phase=202=20CI/test=20s?= =?UTF-8?q?caffolding=20=E2=80=94=20pytest,=20coverage,=20typecheck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes Phase 2 T2.1, T2.2, T2.3, T2.5 of the code-quality roadmap. - evercore-smoke.yml: runs unit-marker pytest with coverage XML upload and a non-blocking ty typecheck step. Both `continue-on-error: true` until tests are explicitly tagged per T2.4. - Makefile: adds test-unit / test-integration / test-e2e / test-cov targets. - tests/conftest.py: auto-tags tests by file path (tests/integration/** → integration; *_e2e.py → e2e; rest → unit). Explicit markers still win. - pytest.ini: fixes section header from [tool:pytest] (legacy setup.cfg form) to [pytest], so pytest actually reads markers and other settings. Registers e2e marker. - pyproject.toml: removes duplicate [tool.pytest.ini_options] section; pytest >= 8 silently ignored it whenever pytest.ini existed. --- .github/workflows/evercore-smoke.yml | 24 ++++++++++++++++ methods/EverCore/Makefile | 26 ++++++++++++++++-- methods/EverCore/pyproject.toml | 13 ++------- methods/EverCore/pytest.ini | 3 +- methods/EverCore/tests/conftest.py | 41 ++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 methods/EverCore/tests/conftest.py diff --git a/.github/workflows/evercore-smoke.yml b/.github/workflows/evercore-smoke.yml index 859fca4e..aa07dda3 100644 --- a/.github/workflows/evercore-smoke.yml +++ b/.github/workflows/evercore-smoke.yml @@ -53,6 +53,30 @@ jobs: PYTHONPATH: src:. run: uv run pytest tests/test_content_item_compat.py -q + # Phase 2 T2.1 — runs pytest in CI. Marker-filtered to "unit" via the + # path heuristic in tests/conftest.py. Many existing tests still + # require databases despite that classification; keep + # continue-on-error until tests are explicitly tagged per T2.4. + - name: Run unit tests with coverage (baseline; non-blocking) + env: + PYTHONPATH: src:. + run: uv run pytest tests/ -m unit --cov=src --cov-report=xml --cov-report=term --maxfail=20 + continue-on-error: true + + - name: Upload coverage artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: evercore-coverage + path: methods/EverCore/coverage.xml + if-no-files-found: warn + + - name: Type check (baseline; warnings only) + env: + PYTHONPATH: src:. + run: uv run ty check --ignore all --error possibly-unresolved-reference + continue-on-error: true + - name: Validate demo payload compatibility env: PYTHONPATH: src:. diff --git a/methods/EverCore/Makefile b/methods/EverCore/Makefile index a45379e3..c0499896 100644 --- a/methods/EverCore/Makefile +++ b/methods/EverCore/Makefile @@ -1,4 +1,4 @@ -.PHONY: dev-setup setup-hooks lint ruff ruff-fix typecheck typecheck-pyright test clean help +.PHONY: dev-setup setup-hooks lint ruff ruff-fix typecheck typecheck-pyright test test-unit test-integration test-e2e test-cov clean help # Default target help: @@ -10,7 +10,11 @@ help: @echo " ruff-fix - Run ruff with --fix and apply formatter" @echo " typecheck - Run ty type checker (default, fast)" @echo " typecheck-pyright - Run pyright type checker (fallback)" - @echo " test - Run tests" + @echo " test - Run all tests" + @echo " test-unit - Run unit tests only (no docker required)" + @echo " test-integration - Run integration tests (needs docker-compose up)" + @echo " test-e2e - Run end-to-end tests" + @echo " test-cov - Run unit tests with coverage report (xml + term)" @echo " clean - Clean up generated files" @echo " help - Show this help message" @@ -71,6 +75,24 @@ typecheck-pyright: test: PYTHONPATH=src uv run pytest tests/ +# Unit tests only. No docker-compose dependencies; safe for CI on every PR. +# Markers are applied automatically in tests/conftest.py based on file path. +test-unit: + PYTHONPATH=src uv run pytest tests/ -m unit + +# Integration tests. Require docker-compose services (Redis, MongoDB, ES, Milvus). +test-integration: + PYTHONPATH=src uv run pytest tests/ -m integration + +# End-to-end tests. Heaviest tier; run on demand or in a nightly job. +test-e2e: + PYTHONPATH=src uv run pytest tests/ -m e2e + +# Coverage report (XML for CI tooling + terminal summary). +test-cov: + PYTHONPATH=src uv run pytest tests/ -m unit \ + --cov=src --cov-report=xml --cov-report=term + # Clean up clean: find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true diff --git a/methods/EverCore/pyproject.toml b/methods/EverCore/pyproject.toml index cd7b9c23..5189ac46 100644 --- a/methods/EverCore/pyproject.toml +++ b/methods/EverCore/pyproject.toml @@ -178,16 +178,9 @@ include = ["src"] # so this first ty rollout stays focused instead of failing on the full baseline. possibly-unresolved-reference = "error" -[tool.pytest.ini_options] -testpaths = ["tests", "unit_test"] -python_files = ["test_*.py", "*_test.py"] -python_classes = ["Test*"] -python_functions = ["test_*"] -addopts = "-v --tb=short" -# Pin pytest-asyncio mode explicitly so existing tests keep their current behavior -# (requires @pytest.mark.asyncio on async tests). Avoids silent behavior changes -# if pytest-asyncio ships a new default in a future release. -asyncio_mode = "strict" +# Pytest config lives in pytest.ini (single source of truth). A duplicate +# section here was silently ignored by pytest >= 8 and only caused a +# rootdir warning. Update pytest.ini for marker / asyncio_mode changes. [dependency-groups] # Development dependencies - for local development diff --git a/methods/EverCore/pytest.ini b/methods/EverCore/pytest.ini index c57fbc0e..5ad216e3 100644 --- a/methods/EverCore/pytest.ini +++ b/methods/EverCore/pytest.ini @@ -1,4 +1,4 @@ -[tool:pytest] +[pytest] # pytest 配置文件 # 测试发现 @@ -28,6 +28,7 @@ markers = memory: 标记为内存相关测试 performance: 标记为性能测试 smoke: 标记为冒烟测试 + e2e: end-to-end tests covering full request flows # 异步测试配置 asyncio_mode = auto diff --git a/methods/EverCore/tests/conftest.py b/methods/EverCore/tests/conftest.py new file mode 100644 index 00000000..51a15f68 --- /dev/null +++ b/methods/EverCore/tests/conftest.py @@ -0,0 +1,41 @@ +"""Pytest configuration shared across the EverCore test suite. + +Auto-marks tests by path so the existing 67 files do not need to be edited +one by one. The marker convention matches the Phase 2 roadmap (T2.3): + +- ``tests/integration/**`` → ``integration`` +- ``tests/*_e2e.py`` → ``e2e`` +- everything else → ``unit`` + +Tests can still apply explicit markers; auto-application is skipped when +any of unit/integration/e2e is already present on the item. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + + +_EXPLICIT = {"unit", "integration", "e2e"} + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + rootpath = Path(config.rootpath).resolve() + for item in items: + if any(m.name in _EXPLICIT for m in item.iter_markers()): + continue + path = Path(item.fspath).resolve() + try: + rel = path.relative_to(rootpath) + except ValueError: + rel = path + parts = rel.parts + name = path.name + if "integration" in parts: + item.add_marker(pytest.mark.integration) + elif name.endswith("_e2e.py"): + item.add_marker(pytest.mark.e2e) + else: + item.add_marker(pytest.mark.unit) From 582ed6fe164ecc2bd1c6bfa95a6486cecd30615a Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:49:36 +0800 Subject: [PATCH 19/24] refactor(EverCore): Phase 3 W5 mechanical cleanup + ruff G/BLE baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes Phase 3 W5 of the code-quality roadmap. Phase 3 W6 (retry refactor) and W7 (catch-all overhaul + custom exception hierarchy) are intentionally not in this commit — both require 24-hour staging soak per roadmap §4.2. Substantive changes - Replace 20 `traceback.print_exc()` call sites with `logger.exception(...)`; remove now-unused `import traceback`. Affected: bootstrap, biz_layer/mem_memorize, mem_db_operations, core/di/scanner, core/component/mongodb_client_factory, core/request/timeout_background, core/oxm/es/migration/utils, and 7 devops_scripts/data_fix scripts. - Consolidate duplicate `RetryConfig` classes. The canonical implementation lives in core/longjob/interfaces (richer API: jitter, backoff_multiplier, retry_on_fatal). The dataclass in core/asynctasks/task_manager is removed and the module re-imports from interfaces. No external consumers, safe merge. Ruff baseline - Enable G (flake8-logging-format) and BLE (flake8-blind-except). - 930 `# noqa: G004` / `# noqa: BLE001` markers applied to existing violations via `ruff check --add-noqa`. New code is held to the rule; baseline shrinks naturally as files are touched. Roadmap status - docs/dev_docs/code_quality_roadmap.md §0 updated to reflect Phase 1 done, Phase 2 partial, Phase 3 W5 done; explicit hand-off note for the W6/W7 boundary. --- .../docs/dev_docs/code_quality_roadmap.md | 20 +- methods/EverCore/pyproject.toml | 6 + .../src/agentic_layer/agentic_utils.py | 30 +-- .../src/agentic_layer/memory_manager.py | 62 +++--- .../agentic_layer/metrics/memorize_metrics.py | 2 +- .../agentic_layer/profile_search_service.py | 8 +- .../src/agentic_layer/rerank_deepinfra.py | 20 +- .../src/agentic_layer/rerank_service.py | 24 +-- .../EverCore/src/agentic_layer/rerank_vllm.py | 20 +- .../src/agentic_layer/retrieval_utils.py | 84 ++++---- .../src/agentic_layer/search_mem_service.py | 8 +- .../src/agentic_layer/vectorize_base.py | 14 +- .../src/agentic_layer/vectorize_service.py | 14 +- .../EverCore/src/api_specs/memory_models.py | 2 +- .../EverCore/src/api_specs/memory_types.py | 8 +- .../src/api_specs/request_converter.py | 8 +- .../src/biz_layer/mem_db_operations.py | 39 ++-- .../EverCore/src/biz_layer/mem_memorize.py | 193 +++++++++--------- methods/EverCore/src/biz_layer/mem_sync.py | 28 +-- methods/EverCore/src/bootstrap.py | 23 +-- methods/EverCore/src/common_utils/cli_ui.py | 2 +- .../src/common_utils/datetime_utils.py | 4 +- .../src/common_utils/url_extractor.py | 10 +- .../addonize/addon_bean_order_strategy.py | 2 +- .../src/core/addons/addons_registry.py | 4 +- .../src/core/asynctasks/task_manager.py | 63 +++--- .../redis_cache_queue/redis_data_processor.py | 4 +- .../redis_length_cache_manager.py | 4 +- .../redis_windows_cache_manager.py | 14 +- .../src/core/component/config_provider.py | 4 +- .../component/elasticsearch_client_factory.py | 8 +- .../core/component/kafka_consumer_factory.py | 10 +- .../core/component/kafka_producer_factory.py | 10 +- .../llm/llm_adapter/anthropic_adapter.py | 4 +- .../llm/llm_adapter/gemini_adapter.py | 4 +- .../llm/llm_adapter/gemini_client.py | 2 +- .../llm/llm_adapter/openai_adapter.py | 6 +- .../core/component/milvus_client_factory.py | 2 +- .../core/component/mongodb_client_factory.py | 16 +- .../src/core/component/redis_provider.py | 24 +-- .../src/core/context/context_manager.py | 10 +- methods/EverCore/src/core/di/container.py | 8 +- methods/EverCore/src/core/di/scanner.py | 21 +- methods/EverCore/src/core/di/utils.py | 2 +- .../src/core/events/event_publisher.py | 24 +-- .../interface/controller/base_controller.py | 4 +- .../controller/debug/debug_controller.py | 36 ++-- .../src/core/lifespan/database_lifespan.py | 2 +- .../core/lifespan/elasticsearch_lifespan.py | 2 +- .../src/core/lifespan/lifespan_factory.py | 4 +- .../src/core/lifespan/longjob_lifespan.py | 2 +- .../src/core/lifespan/metrics_lifespan.py | 2 +- .../src/core/lifespan/milvus_lifespan.py | 2 +- .../src/core/lifespan/mongodb_lifespan.py | 2 +- .../src/core/longjob/longjob_runner.py | 8 +- .../src/core/longjob/recycle_consumer_base.py | 12 +- .../core/middleware/app_logic_middleware.py | 2 +- .../middleware/database_session_middleware.py | 28 +-- .../middleware/hmac_signature_middleware.py | 10 +- .../src/core/middleware/profile_middleware.py | 4 +- .../core/middleware/prometheus_middleware.py | 2 +- .../middleware/sse_exception_middleware.py | 6 +- .../middleware/user_context_middleware.py | 4 +- .../EverCore/src/core/nlp/stopwords_utils.py | 8 +- .../EverCore/src/core/observation/logger.py | 2 +- .../src/core/observation/metrics/gauge.py | 12 +- .../src/core/observation/metrics/server.py | 6 +- .../src/core/oxm/es/base_repository.py | 18 +- .../src/core/oxm/es/migration/utils.py | 6 +- .../src/core/oxm/milvus/base_repository.py | 8 +- .../src/core/oxm/milvus/migration/utils.py | 2 +- .../core/oxm/milvus/milvus_collection_base.py | 20 +- .../src/core/oxm/mongo/base_repository.py | 10 +- .../src/core/oxm/mongo/migration/cli.py | 12 +- .../src/core/oxm/mongo/migration/manager.py | 26 +-- .../EverCore/src/core/oxm/pg/audit_base.py | 2 +- .../kafka_consumer_record_item.py | 4 +- .../src/core/request/app_logic_provider.py | 4 +- .../core/request/request_history_config.py | 2 +- .../core/request/request_history_decorator.py | 22 +- .../src/core/request/timeout_background.py | 10 +- .../src/core/tenants/init_tenant_all.py | 6 +- .../tenants/tenantize/oxm/es/config_utils.py | 2 +- .../oxm/es/tenant_aware_async_document.py | 2 +- .../tenantize/oxm/milvus/config_utils.py | 2 +- .../oxm/milvus/tenant_aware_collection.py | 2 +- .../tenant_aware_collection_with_suffix.py | 14 +- .../tenantize/oxm/mongo/config_utils.py | 2 +- .../oxm/mongo/tenant_aware_client_factory.py | 2 +- .../oxm/mongo/tenant_aware_mongo_client.py | 4 +- .../tenants/tenantize/tenant_cache_utils.py | 2 +- .../data_fix/es_rebuild_index.py | 6 +- .../devops_scripts/data_fix/es_sync_docs.py | 6 +- .../data_fix/es_sync_episodic_memory_docs.py | 6 +- .../data_fix/milvus_rebuild_collection.py | 6 +- .../data_fix/milvus_sync_docs.py | 6 +- .../milvus_sync_episodic_memory_docs.py | 11 +- .../data_fix/mongo_add_timestamp_shard.py | 30 +-- .../src/devops_scripts/i18n/i18n_tool.py | 18 +- .../milvus_admin/browse_collections.py | 10 +- .../sensitive_info/sensitive_info_tool.py | 6 +- .../input/api/memory/group_controller.py | 6 +- .../input/api/memory/memory_controller.py | 22 +- .../input/api/memory/memory_get_controller.py | 4 +- .../api/memory/memory_search_controller.py | 4 +- .../input/api/memory/sender_controller.py | 6 +- .../input/api/memory/settings_controller.py | 4 +- .../repository/agent_case_raw_repository.py | 50 ++--- .../repository/agent_skill_raw_repository.py | 38 ++-- .../atomic_fact_record_raw_repository.py | 12 +- .../conversation_data_raw_repository.py | 8 +- .../conversation_status_raw_repository.py | 8 +- .../episodic_memory_raw_repository.py | 24 +-- .../foresight_record_raw_repository.py | 10 +- .../repository/group_raw_repository.py | 4 +- .../repository/mem_scene_raw_repository.py | 28 +-- .../repository/memcell_raw_repository.py | 46 ++--- .../repository/raw_message_repository.py | 26 +-- .../repository/sender_raw_repository.py | 6 +- .../repository/session_raw_repository.py | 4 +- .../repository/settings_raw_repository.py | 8 +- .../repository/user_profile_raw_repository.py | 42 ++-- .../converter/foresight_milvus_converter.py | 4 +- .../user_profile_milvus_converter.py | 2 +- .../agent_skill_milvus_repository.py | 2 +- .../user_profile_milvus_repository.py | 2 +- .../memory_layer/cluster_manager/manager.py | 34 +-- .../src/memory_layer/llm/openai_provider.py | 4 +- .../conv_memcell_extractor.py | 26 +-- .../memory_extractor/agent_case_extractor.py | 54 ++--- .../memory_extractor/agent_skill_extractor.py | 42 ++-- .../memory_extractor/atomic_fact_extractor.py | 12 +- .../episode_memory_extractor.py | 18 +- .../memory_extractor/foresight_extractor.py | 40 ++-- .../memory_extractor/profile_extractor.py | 32 +-- .../src/memory_layer/memory_manager.py | 14 +- .../profile_indexer/profile_indexer.py | 8 +- .../memory_layer/profile_manager/manager.py | 22 +- methods/EverCore/src/service/group_service.py | 2 +- .../src/service/memcell_delete_service.py | 4 +- .../src/service/raw_message_service.py | 6 +- .../EverCore/src/service/sender_service.py | 2 +- .../EverCore/src/service/session_service.py | 2 +- 143 files changed, 996 insertions(+), 1020 deletions(-) diff --git a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md index 9d26f42e..30e4983d 100644 --- a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md +++ b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md @@ -3,7 +3,25 @@ **Project**: EverCore **Drafted**: 2026-05-26 **Estimated duration**: 6-8 weeks -**Companion document**: [`exception_handling_analysis.md`](./exception_handling_analysis.md) +**Companion documents**: +- [`exception_handling_analysis.md`](./exception_handling_analysis.md) — audit data +- [`slo_definitions.md`](./slo_definitions.md) — SLO/SLI set (Phase 1 T1.5) + +--- + +## 0. Status + +| Phase | State | Notes | +|---|---|---| +| Phase 1: Observability | ✅ **Done** (2026-05-26) | P0 (T1.1 `/livez`+`/readyz`, T1.2 JSON logging, T1.3 typed error metrics) and P1 (T1.4 OpenTelemetry opt-in skeleton, T1.5 SLO doc) all merged. | +| Phase 2: Test foundation | 🟡 **Partial** (2026-05-26) | T2.1 pytest job in CI (non-blocking until tests are explicitly tagged), T2.2 coverage XML + artifact upload, T2.3 path-based test markers + `make test-unit/integration/e2e`, T2.5 typecheck in CI (continue-on-error). T2.4 unit-test backfill and T2.6 benchmark baseline still pending. | +| Phase 3: try-catch cleanup | 🟡 **Week 5 done** (2026-05-26) | W5 mechanical fixes merged: 20× `traceback.print_exc` → `logger.exception`, duplicate `RetryConfig` consolidated (longjob is canonical), `ruff G` + `BLE` rules enabled with baseline noqa markers. W6 (retry refactor) and W7 (catch-all + custom exceptions + large try-block splits) deferred — both require 24h staging soak per §4.2 and are unsafe to bundle into a doc-driven session. | + +**Where this session stopped**: at the Phase 3 W5 / W6 boundary. The +remaining work — Type-A/B/E retry refactor, custom exception hierarchy, +splitting the 241-line try block — depends on test backfill (T2.4) being +real before changes can be validated, and per §4.2 each PR needs a 24h +staging soak. Pick up Phase 3 W6 after T2.4 lands. --- diff --git a/methods/EverCore/pyproject.toml b/methods/EverCore/pyproject.toml index 5189ac46..9bcfb9a3 100644 --- a/methods/EverCore/pyproject.toml +++ b/methods/EverCore/pyproject.toml @@ -132,6 +132,12 @@ select = [ "B", # flake8-bugbear (common bugs) "C4", # flake8-comprehensions "ASYNC", # flake8-async (blocking calls in async, missing awaits) + # Phase 3 W5 baseline rules. Active enforcement holds the line on new + # code via `# noqa: ` markers on existing violations (added + # mechanically with `ruff check --add-noqa`). Removing the noqa as code + # is touched is how the baseline shrinks over time. + "G", # flake8-logging-format (e.g. G004 logging-f-string) + "BLE", # flake8-blind-except (BLE001 bare `except Exception`) ] ignore = [ "E501", # line too long — formatter's job diff --git a/methods/EverCore/src/agentic_layer/agentic_utils.py b/methods/EverCore/src/agentic_layer/agentic_utils.py index 1c7322d3..5af5afec 100644 --- a/methods/EverCore/src/agentic_layer/agentic_utils.py +++ b/methods/EverCore/src/agentic_layer/agentic_utils.py @@ -191,8 +191,8 @@ def parse_json_response(response: str) -> Dict[str, Any]: return result except json.JSONDecodeError as e: - logger.error(f"Failed to parse JSON response: {e}") - logger.debug(f"Raw response: {response[:500]}") + logger.error(f"Failed to parse JSON response: {e}") # noqa: G004 + logger.debug(f"Raw response: {response[:500]}") # noqa: G004 raise ValueError(f"JSON parse error: {e}") @@ -223,8 +223,8 @@ def parse_sufficiency_response(response: str) -> Tuple[bool, str, List[str]]: return is_sufficient, reasoning, missing_info - except Exception as e: - logger.error(f"Failed to parse sufficiency response: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to parse sufficiency response: {e}") # noqa: G004 # Conservative fallback: assume sufficient return True, f"Parse error: {str(e)}", [] @@ -268,11 +268,11 @@ def parse_multi_query_response( # Limit to maximum 3 queries valid_queries = valid_queries[:3] - logger.info(f"Generated {len(valid_queries)} valid queries") + logger.info(f"Generated {len(valid_queries)} valid queries") # noqa: G004 return valid_queries, reasoning - except Exception as e: - logger.error(f"Failed to parse multi-query response: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to parse multi-query response: {e}") # noqa: G004 # Fallback: return original query return [original_query], f"Parse error: {str(e)}" @@ -320,7 +320,7 @@ async def check_sufficiency( ) # 3. Call LLM - logger.debug(f"Calling LLM for sufficiency check on query: {query[:50]}...") + logger.debug(f"Calling LLM for sufficiency check on query: {query[:50]}...") # noqa: G004 result_text = await llm_provider.generate( prompt=prompt, temperature=0.0, # Low temperature for more stable judgment @@ -330,8 +330,8 @@ async def check_sufficiency( # 4. Parse response is_sufficient, reasoning, missing_info = parse_sufficiency_response(result_text) - logger.info(f"Sufficiency check result: {is_sufficient}") - logger.debug(f"Reasoning: {reasoning}") + logger.info(f"Sufficiency check result: {is_sufficient}") # noqa: G004 + logger.debug(f"Reasoning: {reasoning}") # noqa: G004 return is_sufficient, reasoning, missing_info @@ -341,7 +341,7 @@ async def check_sufficiency( return True, "Timeout: LLM took too long", [] except Exception as e: - logger.error(f"Sufficiency check failed: {e}", exc_info=True) + logger.error(f"Sufficiency check failed: {e}", exc_info=True) # noqa: G004, G201 # Conservative fallback: assume sufficient return True, f"Error: {str(e)}", [] @@ -396,7 +396,7 @@ async def generate_multi_queries( ) # 3. Call LLM - logger.debug(f"Generating multi-queries for: {original_query[:50]}...") + logger.debug(f"Generating multi-queries for: {original_query[:50]}...") # noqa: G004 result_text = await llm_provider.generate( prompt=prompt, temperature=0.4, # Slightly higher temperature to increase query diversity @@ -406,9 +406,9 @@ async def generate_multi_queries( # 4. Parse response queries, reasoning = parse_multi_query_response(result_text, original_query) - logger.info(f"Generated {len(queries)} queries") + logger.info(f"Generated {len(queries)} queries") # noqa: G004 for i, q in enumerate(queries, 1): - logger.debug(f" Query {i}: {q[:80]}{'...' if len(q) > 80 else ''}") + logger.debug(f" Query {i}: {q[:80]}{'...' if len(q) > 80 else ''}") # noqa: G004 return queries, reasoning @@ -418,6 +418,6 @@ async def generate_multi_queries( return [original_query], "Timeout: used original query" except Exception as e: - logger.error(f"Multi-query generation failed: {e}", exc_info=True) + logger.error(f"Multi-query generation failed: {e}", exc_info=True) # noqa: G004, G201 # Fallback to original query return [original_query], f"Error: {str(e)}" diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index 96f05a6e..ee56d980 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -217,7 +217,7 @@ async def retrieve_mem( retrieve_method = retrieve_mem_request.retrieve_method logger.info( - f"retrieve_mem dispatching request: user_id={retrieve_mem_request.user_id}, " + f"retrieve_mem dispatching request: user_id={retrieve_mem_request.user_id}, " # noqa: G004 f"retrieve_method={retrieve_method}, query={retrieve_mem_request.query}, " f"search_profile={search_profile}, non_profile_types={[t.value for t in non_profile_types]}" ) @@ -296,7 +296,7 @@ async def retrieve_mem( return response except Exception as e: - logger.error(f"Error in retrieve_mem: {e}", exc_info=True) + logger.error(f"Error in retrieve_mem: {e}", exc_info=True) # noqa: G004, G201 return RetrieveMemResponse( profiles=[], memories=[], @@ -358,11 +358,11 @@ async def _search_profiles( ) profiles.append(profile_item) - logger.debug(f"Profile search returned {len(profiles)} items") + logger.debug(f"Profile search returned {len(profiles)} items") # noqa: G004 return profiles except Exception as e: - logger.error(f"Error in _search_profiles: {e}", exc_info=True) + logger.error(f"Error in _search_profiles: {e}", exc_info=True) # noqa: G004, G201 return [] def _build_combined_response( @@ -429,12 +429,12 @@ async def _get_pending_messages( ) logger.debug( - f"Retrieved {len(result)} pending messages: " + f"Retrieved {len(result)} pending messages: " # noqa: G004 f"user_id={user_id}, group_ids={group_ids}" ) return result except Exception as e: - logger.error(f"Error fetching pending messages: {e}", exc_info=True) + logger.error(f"Error fetching pending messages: {e}", exc_info=True) # noqa: G004, G201 return [] # Keyword retrieval method (original retrieve_mem logic) @@ -463,7 +463,7 @@ async def retrieve_mem_keyword( return await self._to_response(hits, retrieve_mem_request) except Exception as e: - logger.error(f"Error in retrieve_mem_keyword: {e}", exc_info=True) + logger.error(f"Error in retrieve_mem_keyword: {e}", exc_info=True) # noqa: G004, G201 return await self._to_response([], retrieve_mem_request) async def get_keyword_search_results( @@ -507,7 +507,7 @@ async def get_keyword_search_results( else: query_words = [] - logger.debug(f"query_words: {query_words}") + logger.debug(f"query_words: {query_words}") # noqa: G004 # Build time range filter conditions, handle None values date_range = {} @@ -520,11 +520,11 @@ async def get_keyword_search_results( repo_class = ES_REPO_MAP.get(mem_type) if not repo_class: - logger.warning(f"Unsupported memory_type: {mem_type}") + logger.warning(f"Unsupported memory_type: {mem_type}") # noqa: G004 return [] es_repo = get_bean_by_type(repo_class) - logger.debug(f"Using {repo_class.__name__} for {mem_type}") + logger.debug(f"Using {repo_class.__name__} for {mem_type}") # noqa: G004 results = await es_repo.multi_search( query=query_words, @@ -564,7 +564,7 @@ async def get_keyword_search_results( stage=RetrieveMethod.KEYWORD.value, error_type=self._classify_retrieve_error(e), ) - logger.error(f"Error in get_keyword_search_results: {e}") + logger.error(f"Error in get_keyword_search_results: {e}") # noqa: G004 raise # Vector-based memory retrieval @@ -592,8 +592,8 @@ async def retrieve_mem_vector( hits = hits[:top_k] return await self._to_response(hits, retrieve_mem_request) - except Exception as e: - logger.error(f"Error in retrieve_mem_vector: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error in retrieve_mem_vector: {e}") # noqa: G004 return await self._to_response([], retrieve_mem_request) async def get_vector_search_results( @@ -617,7 +617,7 @@ async def get_vector_search_results( try: # Get parameters from Request logger.debug( - f"get_vector_search_results called with retrieve_mem_request: {retrieve_mem_request}" + f"get_vector_search_results called with retrieve_mem_request: {retrieve_mem_request}" # noqa: G004 ) if not retrieve_mem_request: raise ValueError( @@ -644,14 +644,14 @@ async def get_vector_search_results( mem_type = retrieve_mem_request.memory_types[0] logger.debug( - f"retrieve_mem_vector called with query: {query}, user_id: {user_id}, group_ids: {group_ids}, top_k: {top_k}" + f"retrieve_mem_vector called with query: {query}, user_id: {user_id}, group_ids: {group_ids}, top_k: {top_k}" # noqa: G004 ) # Get vectorization service vectorize_service = get_vectorize_service() # Convert query text to vector (embedding stage) - logger.debug(f"Starting to vectorize query text: {query}") + logger.debug(f"Starting to vectorize query text: {query}") # noqa: G004 embedding_start = time.perf_counter() query_vector = await vectorize_service.get_embedding(query) query_vector_list = query_vector.tolist() # Convert to list format @@ -662,7 +662,7 @@ async def get_vector_search_results( duration_seconds=time.perf_counter() - embedding_start, ) logger.debug( - f"Query text vectorization completed, vector dimension: {len(query_vector_list)}" + f"Query text vectorization completed, vector dimension: {len(query_vector_list)}" # noqa: G004 ) # Select Milvus repository based on memory type @@ -780,7 +780,7 @@ async def get_vector_search_results( stage=RetrieveMethod.VECTOR.value, error_type=self._classify_retrieve_error(e), ) - logger.error(f"Error in get_vector_search_results: {e}") + logger.error(f"Error in get_vector_search_results: {e}") # noqa: G004 raise # Hybrid memory retrieval @@ -800,8 +800,8 @@ async def retrieve_mem_hybrid( retrieve_mem_request, retrieve_method=RetrieveMethod.HYBRID.value ) return await self._to_response(hits, retrieve_mem_request) - except Exception as e: - logger.error(f"Error in retrieve_mem_hybrid: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error in retrieve_mem_hybrid: {e}") # noqa: G004 return await self._to_response([], retrieve_mem_request) # ================== Core Internal Methods ================== @@ -851,7 +851,7 @@ async def _rerank( filtered_count = original_count - len(result) if filtered_count > 0: logger.debug( - f"Rerank threshold filtering: {filtered_count} docs filtered " + f"Rerank threshold filtering: {filtered_count} docs filtered " # noqa: G004 f"(threshold={DEFAULT_RERANK_SCORE_THRESHOLD})" ) @@ -987,7 +987,7 @@ async def retrieve_mem_agentic( try: llm_provider = build_default_provider() - logger.info(f"Agentic Retrieval: {req.query[:60]}...") + logger.info(f"Agentic Retrieval: {req.query[:60]}...") # noqa: G004 # ========== Round 1: Hybrid search ========== req1 = RetrieveMemRequest( @@ -998,7 +998,7 @@ async def retrieve_mem_agentic( memory_types=req.memory_types, ) round1 = await self._search_hybrid(req1, retrieve_method='agentic') - logger.info(f"Round 1: {len(round1)} memories") + logger.info(f"Round 1: {len(round1)} memories") # noqa: G004 if not round1: return await self._to_response([], req) @@ -1031,7 +1031,7 @@ async def retrieve_mem_agentic( max_docs=config.round1_rerank_top_n, ) logger.info( - f"LLM: {'Sufficient' if is_sufficient else 'Insufficient'} - {reasoning}" + f"LLM: {'Sufficient' if is_sufficient else 'Insufficient'} - {reasoning}" # noqa: G004 ) if is_sufficient: @@ -1049,7 +1049,7 @@ async def retrieve_mem_agentic( max_docs=config.round1_rerank_top_n, num_queries=config.num_queries, ) - logger.info(f"Generated {len(refined_queries)} queries") + logger.info(f"Generated {len(refined_queries)} queries") # noqa: G004 # Parallel hybrid search async def do_search(q: str) -> List[Dict]: @@ -1079,7 +1079,7 @@ async def do_search(q: str) -> List[Dict]: seen_ids = {m.get("id") for m in round1} round2_unique = [m for m in all_round2 if m.get("id") not in seen_ids] combined = round1 + round2_unique[: config.combined_total - len(round1)] - logger.info(f"Combined: {len(combined)} memories") + logger.info(f"Combined: {len(combined)} memories") # noqa: G004 # ========== Final Rerank ========== # Calculate final rerank quantity: satisfy both config (40) and user request (top_k) @@ -1102,7 +1102,7 @@ async def do_search(q: str) -> List[Dict]: return await self._to_response(final_results, req) except Exception as e: - logger.error(f"Error in retrieve_mem_agentic: {e}", exc_info=True) + logger.error(f"Error in retrieve_mem_agentic: {e}", exc_info=True) # noqa: G004, G201 return await self._to_response([], req) async def _batch_get_memcells( @@ -1123,7 +1123,7 @@ async def _batch_get_memcells( # Deduplicate event_ids unique_event_ids = list(set(event_ids)) logger.debug( - f"Batch get MemCells: Total {len(unique_event_ids)} (before deduplication: {len(event_ids)})" + f"Batch get MemCells: Total {len(unique_event_ids)} (before deduplication: {len(event_ids)})" # noqa: G004 ) memcell_repo = get_bean_by_type(MemCellRawRepository) @@ -1133,14 +1133,14 @@ async def _batch_get_memcells( for i in range(0, len(unique_event_ids), batch_size): batch_event_ids = unique_event_ids[i : i + batch_size] logger.debug( - f"Getting batch {i // batch_size + 1} MemCells: {len(batch_event_ids)} items" + f"Getting batch {i // batch_size + 1} MemCells: {len(batch_event_ids)} items" # noqa: G004 ) batch_memcells = await memcell_repo.get_by_event_ids(batch_event_ids) all_memcells.update(batch_memcells) logger.debug( - f"Batch get MemCells completed: Successfully retrieved {len(all_memcells)} items" + f"Batch get MemCells completed: Successfully retrieved {len(all_memcells)} items" # noqa: G004 ) return all_memcells @@ -1313,7 +1313,7 @@ async def group_by_groupid_stratagy( if memcell and memcell.original_data: original_data = memcell.original_data else: - logger.debug(f"Memcell not found: event_id={parent_id}") + logger.debug(f"Memcell not found: event_id={parent_id}") # noqa: G004 # Create object based on memory type base_kwargs = { diff --git a/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py b/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py index 64b11480..8508a1b6 100644 --- a/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py +++ b/methods/EverCore/src/agentic_layer/metrics/memorize_metrics.py @@ -56,7 +56,7 @@ def get_space_id_for_metrics() -> str: tenant = get_current_tenant() if tenant and tenant.tenant_detail and tenant.tenant_detail.tenant_info: return tenant.tenant_detail.tenant_info.get('space_id', 'default') - except Exception: + except Exception: # noqa: BLE001 pass return 'default' diff --git a/methods/EverCore/src/agentic_layer/profile_search_service.py b/methods/EverCore/src/agentic_layer/profile_search_service.py index 80efe058..1b073292 100644 --- a/methods/EverCore/src/agentic_layer/profile_search_service.py +++ b/methods/EverCore/src/agentic_layer/profile_search_service.py @@ -148,7 +148,7 @@ async def search_profiles( recall_limit = top_k * 2 if top_k > 0 else PROFILE_DEFAULT_TOPK * 2 logger.info( - f"🔍 Profile search params: user_id={user_id}, group_id={group_id}, " + f"🔍 Profile search params: user_id={user_id}, group_id={group_id}, " # noqa: G004 f"top_k={top_k}, recall_limit={recall_limit}, score_threshold={score_threshold}" ) @@ -161,7 +161,7 @@ async def search_profiles( ) logger.info( - f"✅ Milvus returned {len(milvus_results)} results, will take top {top_k}" + f"✅ Milvus returned {len(milvus_results)} results, will take top {top_k}" # noqa: G004 ) # Step 3: Process results - parse embed_text and format output @@ -198,7 +198,7 @@ async def search_profiles( # Log profile scores for debugging if profiles: scores_str = ", ".join([f"{p['score']:.4f}" for p in profiles]) - logger.info(f"📊 Profile scores: [{scores_str}]") + logger.info(f"📊 Profile scores: [{scores_str}]") # noqa: G004 logger.info( "Profile search completed: user_id=%s, group_id=%s, " @@ -213,7 +213,7 @@ async def search_profiles( return result except Exception as e: - logger.error( + logger.error( # noqa: G201 "Profile search failed: user_id=%s, group_id=%s, error=%s", user_id, group_id, diff --git a/methods/EverCore/src/agentic_layer/rerank_deepinfra.py b/methods/EverCore/src/agentic_layer/rerank_deepinfra.py index 9cafb88b..80ec6002 100644 --- a/methods/EverCore/src/agentic_layer/rerank_deepinfra.py +++ b/methods/EverCore/src/agentic_layer/rerank_deepinfra.py @@ -44,7 +44,7 @@ def __init__(self, config: Optional[DeepInfraRerankConfig] = None): self.config = config self.session: Optional[aiohttp.ClientSession] = None self._semaphore = asyncio.Semaphore(config.max_concurrent_requests) - logger.info(f"Initialized DeepInfraRerankService | model={config.model}") + logger.info(f"Initialized DeepInfraRerankService | model={config.model}") # noqa: G004 async def _ensure_session(self): """Ensure HTTP session is created""" @@ -110,7 +110,7 @@ async def _send_rerank_request_batch( else: error_text = await response.text() logger.error( - f"DeepInfra rerank API error {response.status}: {error_text}" + f"DeepInfra rerank API error {response.status}: {error_text}" # noqa: G004 ) if attempt < self.config.max_retries - 1: await asyncio.sleep(2**attempt) @@ -118,8 +118,8 @@ async def _send_rerank_request_batch( raise RerankError( f"API failed: {response.status} - {error_text}" ) - except Exception as e: - logger.error(f"DeepInfra rerank exception: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"DeepInfra rerank exception: {e}") # noqa: G004 if attempt < self.config.max_retries - 1: await asyncio.sleep(2**attempt) continue @@ -182,7 +182,7 @@ async def rerank_documents( for i, result in enumerate(batch_results): if isinstance(result, Exception): - logger.error(f"Rerank batch {i} failed: {result}") + logger.error(f"Rerank batch {i} failed: {result}") # noqa: G004 batch_len = len(batches[i]) all_scores.extend([-100.0] * batch_len) continue @@ -196,7 +196,7 @@ async def rerank_documents( try: collector = get_bean_by_type(TokenUsageCollector) collector.add(self.config.model, total_input_tokens, 0, call_type="rerank") - except Exception: + except Exception: # noqa: BLE001 pass combined_response = { @@ -262,7 +262,7 @@ async def rerank_memories( # Call reranking API try: logger.debug( - f"Starting reranking, query text: {query}, number of texts: {len(all_texts)}" + f"Starting reranking, query text: {query}, number of texts: {len(all_texts)}" # noqa: G004 ) rerank_result = await self.rerank_documents(query, all_texts, instruction) @@ -290,12 +290,12 @@ async def rerank_memories( if reranked_hits: top_scores = [f"{h.get('score', 0):.4f}" for h in reranked_hits[:3]] logger.info( - f"Reranking completed: {len(reranked_hits)} results, top scores: {top_scores}" + f"Reranking completed: {len(reranked_hits)} results, top scores: {top_scores}" # noqa: G004 ) return reranked_hits - except Exception as e: - logger.error(f"Error during reranking: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error during reranking: {e}") # noqa: G004 # If reranking fails, return original results (sorted by original score) sorted_hits = sorted(hits, key=lambda x: x.get('score', 0), reverse=True) if top_k is not None and top_k > 0: diff --git a/methods/EverCore/src/agentic_layer/rerank_service.py b/methods/EverCore/src/agentic_layer/rerank_service.py index 251cff50..2319472c 100644 --- a/methods/EverCore/src/agentic_layer/rerank_service.py +++ b/methods/EverCore/src/agentic_layer/rerank_service.py @@ -231,7 +231,7 @@ def __init__(self, config: Optional[HybridRerankConfig] = None): ) logger.info( - f"Initialized HybridRerankService | " + f"Initialized HybridRerankService | " # noqa: G004 f"primary={config.primary_provider} | " f"fallback={config.fallback_provider} | " f"fallback_enabled={config.enable_fallback} | " @@ -264,7 +264,7 @@ def _log_rerank_input( logger.info( "Rerank input: %d docs, %d tokens", documents_count, total_tokens ) - except Exception: + except Exception: # noqa: BLE001 logger.info( "Rerank input: %d docs (token count unavailable)", documents_count ) @@ -379,7 +379,7 @@ async def execute_with_fallback( > self.config.failure_reset_interval ): logger.info( - f"🔄 Resetting failure count ({self.config._primary_failure_count}) after " + f"🔄 Resetting failure count ({self.config._primary_failure_count}) after " # noqa: G004 f"{int(current_time - self.config._last_failure_time)}s of no failures" ) self.config._primary_failure_count = 0 @@ -392,7 +392,7 @@ async def execute_with_fallback( and self.config._primary_failure_count >= self.config.max_primary_failures ): logger.info( - f"⚠️ Primary service has {self.config._primary_failure_count} failures " + f"⚠️ Primary service has {self.config._primary_failure_count} failures " # noqa: G004 f"(>= {self.config.max_primary_failures}), skipping and using {self.config.fallback_provider} directly" ) @@ -407,8 +407,8 @@ async def execute_with_fallback( result = await fallback_func() return result - except Exception as fallback_error: - logger.error(f"❌ Fallback service also failed: {fallback_error}") + except Exception as fallback_error: # noqa: BLE001 + logger.error(f"❌ Fallback service also failed: {fallback_error}") # noqa: G004 # Record fallback error fallback_error_type = self._classify_error(fallback_error) @@ -426,13 +426,13 @@ async def execute_with_fallback( self.config._primary_failure_count = 0 return result - except Exception as primary_error: + except Exception as primary_error: # noqa: BLE001 # Increment failure count and update timestamp self.config._primary_failure_count += 1 self.config._last_failure_time = time.time() logger.warning( - f"Primary service ({self.config.primary_provider}) {operation_name} failed " + f"Primary service ({self.config.primary_provider}) {operation_name} failed " # noqa: G004 f"(count: {self.config._primary_failure_count}): {primary_error}" ) @@ -454,14 +454,14 @@ async def execute_with_fallback( if self.config._primary_failure_count >= self.config.max_primary_failures: fallback_reason = 'max_failures_exceeded' logger.warning( - f"⚠️ Primary service exceeded max failures ({self.config.max_primary_failures}), " + f"⚠️ Primary service exceeded max failures ({self.config.max_primary_failures}), " # noqa: G004 f"using {self.config.fallback_provider} fallback" ) # Try fallback service try: logger.info( - f"🔄 Falling back to {self.config.fallback_provider} for {operation_name}" + f"🔄 Falling back to {self.config.fallback_provider} for {operation_name}" # noqa: G004 ) # Record fallback event @@ -474,8 +474,8 @@ async def execute_with_fallback( result = await fallback_func() return result - except Exception as fallback_error: - logger.error(f"❌ Fallback also failed: {fallback_error}") + except Exception as fallback_error: # noqa: BLE001 + logger.error(f"❌ Fallback also failed: {fallback_error}") # noqa: G004 # Record fallback error fallback_error_type = self._classify_error(fallback_error) diff --git a/methods/EverCore/src/agentic_layer/rerank_vllm.py b/methods/EverCore/src/agentic_layer/rerank_vllm.py index 6545b50f..e4d3ba79 100644 --- a/methods/EverCore/src/agentic_layer/rerank_vllm.py +++ b/methods/EverCore/src/agentic_layer/rerank_vllm.py @@ -45,7 +45,7 @@ def __init__(self, config: Optional[VllmRerankConfig] = None): self.session: Optional[aiohttp.ClientSession] = None self._semaphore = asyncio.Semaphore(config.max_concurrent_requests) logger.info( - f"Initialized VllmRerankService | url={config.base_url} | model={config.model}" # skip-sensitive-check + f"Initialized VllmRerankService | url={config.base_url} | model={config.model}" # skip-sensitive-check # noqa: G004 ) async def _ensure_session(self): @@ -117,7 +117,7 @@ async def _send_rerank_request_batch( else: error_text = await response.text() logger.warning( - f"vLLM rerank API error (status {response.status}, attempt {attempt + 1}/{self.config.max_retries}): {error_text}" + f"vLLM rerank API error (status {response.status}, attempt {attempt + 1}/{self.config.max_retries}): {error_text}" # noqa: G004 ) if attempt < self.config.max_retries - 1: await asyncio.sleep(2**attempt) @@ -128,7 +128,7 @@ async def _send_rerank_request_batch( ) except asyncio.TimeoutError: logger.warning( - f"vLLM rerank timeout (attempt {attempt + 1}/{self.config.max_retries}), timeout={self.config.timeout}s" + f"vLLM rerank timeout (attempt {attempt + 1}/{self.config.max_retries}), timeout={self.config.timeout}s" # noqa: G004 ) if attempt < self.config.max_retries - 1: await asyncio.sleep(2**attempt) @@ -139,7 +139,7 @@ async def _send_rerank_request_batch( ) except aiohttp.ClientError as e: logger.warning( - f"vLLM rerank client error (attempt {attempt + 1}/{self.config.max_retries}): {e}" + f"vLLM rerank client error (attempt {attempt + 1}/{self.config.max_retries}): {e}" # noqa: G004 ) if attempt < self.config.max_retries - 1: await asyncio.sleep(2**attempt) @@ -148,8 +148,8 @@ async def _send_rerank_request_batch( raise RerankError( f"Rerank request failed after {self.config.max_retries} attempts: {e}" ) - except Exception as e: - logger.error(f"Unexpected error in vLLM rerank request: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Unexpected error in vLLM rerank request: {e}") # noqa: G004 raise RerankError(f"Unexpected rerank error: {e}") async def rerank_documents( @@ -189,7 +189,7 @@ async def rerank_documents( all_scores = [] for i, result in enumerate(batch_results): if isinstance(result, Exception): - logger.error(f"Rerank batch {i} failed: {result}") + logger.error(f"Rerank batch {i} failed: {result}") # noqa: G004 batch_len = len(batches[i]) all_scores.extend([-100.0] * batch_len) continue @@ -204,7 +204,7 @@ async def rerank_documents( try: collector = get_bean_by_type(TokenUsageCollector) collector.add(self.config.model, 0, 0, call_type="rerank") - except Exception: + except Exception: # noqa: BLE001 pass # Convert to same format as DeepInfra @@ -297,14 +297,14 @@ async def rerank_memories( if reranked_hits: top_scores = [f"{h.get('score', 0):.4f}" for h in reranked_hits[:3]] logger.info( - f"Reranked {len(hits)} hits -> {len(reranked_hits)} results, " + f"Reranked {len(hits)} hits -> {len(reranked_hits)} results, " # noqa: G004 f"top scores: {top_scores}" ) return reranked_hits except Exception as e: - logger.error(f"Error in rerank_memories: {e}") + logger.error(f"Error in rerank_memories: {e}") # noqa: G004 # Re-raise exception to allow HybridRerankService fallback raise diff --git a/methods/EverCore/src/agentic_layer/retrieval_utils.py b/methods/EverCore/src/agentic_layer/retrieval_utils.py index 2f2a6282..9fde9f27 100644 --- a/methods/EverCore/src/agentic_layer/retrieval_utils.py +++ b/methods/EverCore/src/agentic_layer/retrieval_utils.py @@ -239,11 +239,11 @@ async def lightweight_retrieval( if doc_norm > 0: sim = np.dot(query_vec, doc_vec) / (query_norm * doc_norm) scores.append((mem, float(sim))) - except Exception: + except Exception: # noqa: BLE001 continue emb_results = sorted(scores, key=lambda x: x[1], reverse=True)[:emb_top_n] - except Exception: + except Exception: # noqa: BLE001 pass metadata["emb_count"] = len(emb_results) @@ -397,7 +397,7 @@ async def multi_query_retrieval( metadata["total_latency_ms"] = (time.time() - start_time) * 1000 return [], metadata - logger.info(f"Executing {len(queries)} queries in parallel...") + logger.info(f"Executing {len(queries)} queries in parallel...") # noqa: G004 # Execute hybrid retrieval for all queries in parallel tasks = [ @@ -414,7 +414,7 @@ async def multi_query_retrieval( valid_results = [] for i, result in enumerate(multi_query_results, 1): if isinstance(result, Exception): - logger.error(f"Query {i} failed: {result}") + logger.error(f"Query {i} failed: {result}") # noqa: G004 continue results, query_metadata = result @@ -427,7 +427,7 @@ async def multi_query_retrieval( "latency_ms": query_metadata.get("total_latency_ms", 0), } ) - logger.debug(f"Query {i}: Retrieved {len(results)} documents") + logger.debug(f"Query {i}: Retrieved {len(results)} documents") # noqa: G004 if not valid_results: logger.warning("All queries failed") @@ -438,7 +438,7 @@ async def multi_query_retrieval( metadata["total_docs_before_fusion"] = sum(len(r) for r in valid_results) # Use multi-query RRF fusion - logger.info(f"Fusing {len(valid_results)} query results...") + logger.info(f"Fusing {len(valid_results)} query results...") # noqa: G004 fused_results = multi_rrf_fusion(valid_results, k=rrf_k) # Take Top-N @@ -448,7 +448,7 @@ async def multi_query_retrieval( metadata["total_latency_ms"] = (time.time() - start_time) * 1000 logger.info( - f"Multi-query retrieval: {metadata['total_docs_before_fusion']} → {len(final_results)} docs" + f"Multi-query retrieval: {metadata['total_docs_before_fusion']} → {len(final_results)} docs" # noqa: G004 ) return final_results, metadata @@ -480,7 +480,7 @@ async def rerank_candidates( try: logger.debug( - f"Reranking {len(candidates)} candidates for query: {query[:50]}..." + f"Reranking {len(candidates)} candidates for query: {query[:50]}..." # noqa: G004 ) # 🔥 Convert format: transform [(doc, score)] to format expected by rerank service @@ -531,21 +531,21 @@ async def rerank_candidates( new_score = hit.get("score", 0.0) else: # If returned is tuple, format is wrong, skip - logger.warning(f"Unexpected rerank result type: {type(hit)}") + logger.warning(f"Unexpected rerank result type: {type(hit)}") # noqa: G004 continue if 0 <= idx < len(candidates): doc = candidates[idx][0] reranked_results.append((doc, new_score)) - logger.debug(f"Rerank complete: {len(reranked_results)} results") + logger.debug(f"Rerank complete: {len(reranked_results)} results") # noqa: G004 return reranked_results if reranked_results else candidates[:top_n] else: logger.warning("Rerank returned empty results, using original") return candidates[:top_n] except Exception as e: - logger.error(f"Rerank failed: {e}, using original ranking", exc_info=True) + logger.error(f"Rerank failed: {e}, using original ranking", exc_info=True) # noqa: G004, G201 return candidates[:top_n] @@ -614,9 +614,9 @@ async def agentic_retrieval( "total_latency_ms": 0.0, } - logger.info(f"{'=' * 60}") - logger.info(f"Agentic Retrieval: {query[:60]}...") - logger.info(f"{'=' * 60}") + logger.info(f"{'=' * 60}") # noqa: G004 + logger.info(f"Agentic Retrieval: {query[:60]}...") # noqa: G004 + logger.info(f"{'=' * 60}") # noqa: G004 # ========== Round 1: Hybrid search Top 20 ========== logger.info("Round 1: Hybrid search for Top 20...") @@ -633,15 +633,15 @@ async def agentic_retrieval( metadata["round1_count"] = len(round1_results) metadata["round1_latency_ms"] = round1_metadata.get("total_latency_ms", 0) - logger.info(f"Round 1: Retrieved {len(round1_results)} documents") + logger.info(f"Round 1: Retrieved {len(round1_results)} documents") # noqa: G004 if not round1_results: logger.warning("Round 1 returned no results") metadata["total_latency_ms"] = (time.time() - start_time) * 1000 return [], metadata - except Exception as e: - logger.error(f"Round 1 failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Round 1 failed: {e}") # noqa: G004 metadata["total_latency_ms"] = (time.time() - start_time) * 1000 return [], metadata @@ -659,10 +659,10 @@ async def agentic_retrieval( ) metadata["round1_reranked_count"] = len(reranked_top5) - logger.info(f"Rerank: Got Top {len(reranked_top5)} for sufficiency check") + logger.info(f"Rerank: Got Top {len(reranked_top5)} for sufficiency check") # noqa: G004 - except Exception as e: - logger.error(f"Rerank failed: {e}, using original Top 10") + except Exception as e: # noqa: BLE001 + logger.error(f"Rerank failed: {e}, using original Top 10") # noqa: G004 reranked_top5 = round1_results[: config.round1_rerank_top_n] metadata["round1_reranked_count"] = len(reranked_top5) else: @@ -692,12 +692,12 @@ async def agentic_retrieval( metadata["missing_info"] = missing_info logger.info( - f"LLM Result: {'✅ Sufficient' if is_sufficient else '❌ Insufficient'}" + f"LLM Result: {'✅ Sufficient' if is_sufficient else '❌ Insufficient'}" # noqa: G004 ) - logger.info(f"LLM Reasoning: {reasoning}") + logger.info(f"LLM Reasoning: {reasoning}") # noqa: G004 - except Exception as e: - logger.error(f"Sufficiency check failed: {e}, assuming sufficient") + except Exception as e: # noqa: BLE001 + logger.error(f"Sufficiency check failed: {e}, assuming sufficient") # noqa: G004 metadata["total_latency_ms"] = (time.time() - start_time) * 1000 return round1_results, metadata @@ -709,14 +709,14 @@ async def agentic_retrieval( metadata["final_count"] = len(final_results) metadata["total_latency_ms"] = (time.time() - start_time) * 1000 - logger.info(f"Complete: Latency {metadata['total_latency_ms']:.0f}ms") + logger.info(f"Complete: Latency {metadata['total_latency_ms']:.0f}ms") # noqa: G004 return final_results, metadata # ========== If insufficient: enter Round 2 ========== metadata["is_multi_round"] = True logger.info("Decision: Insufficient, entering Round 2") if missing_info: - logger.info(f"Missing: {', '.join(missing_info)}") + logger.info(f"Missing: {', '.join(missing_info)}") # noqa: G004 # ========== LLM generate multiple refined queries ========== if config.enable_multi_query: @@ -736,12 +736,12 @@ async def agentic_retrieval( metadata["query_strategy"] = query_strategy metadata["num_queries"] = len(refined_queries) - logger.info(f"Generated {len(refined_queries)} queries") + logger.info(f"Generated {len(refined_queries)} queries") # noqa: G004 for i, q in enumerate(refined_queries, 1): - logger.debug(f" Query {i}: {q[:80]}...") + logger.debug(f" Query {i}: {q[:80]}...") # noqa: G004 - except Exception as e: - logger.error(f"Query generation failed: {e}, using original query") + except Exception as e: # noqa: BLE001 + logger.error(f"Query generation failed: {e}, using original query") # noqa: G004 refined_queries = [query] metadata["refined_queries"] = refined_queries metadata["num_queries"] = 1 @@ -752,7 +752,7 @@ async def agentic_retrieval( metadata["num_queries"] = 1 # ========== Round 2: Execute multiple queries retrieval in parallel ========== - logger.info(f"Round 2: Executing {len(refined_queries)} queries in parallel...") + logger.info(f"Round 2: Executing {len(refined_queries)} queries in parallel...") # noqa: G004 try: round2_results, round2_metadata = await multi_query_retrieval( @@ -770,10 +770,10 @@ async def agentic_retrieval( "total_docs_before_fusion", 0 ) - logger.info(f"Round 2: Retrieved {len(round2_results)} unique documents") + logger.info(f"Round 2: Retrieved {len(round2_results)} unique documents") # noqa: G004 - except Exception as e: - logger.error(f"Round 2 failed: {e}, using Round 1 results") + except Exception as e: # noqa: BLE001 + logger.error(f"Round 2 failed: {e}, using Round 1 results") # noqa: G004 metadata["total_latency_ms"] = (time.time() - start_time) * 1000 return round1_results, metadata @@ -792,12 +792,12 @@ async def agentic_retrieval( combined_results.extend(round2_unique[:needed_from_round2]) logger.info( - f"Merge: Round1={len(round1_results)}, Round2_unique={len(round2_unique[:needed_from_round2])}, Total={len(combined_results)}" + f"Merge: Round1={len(round1_results)}, Round2_unique={len(round2_unique[:needed_from_round2])}, Total={len(combined_results)}" # noqa: G004 ) # ========== Rerank merged documents ========== if config.use_reranker and len(combined_results) > 0: - logger.info(f"Rerank: Reranking {len(combined_results)} documents...") + logger.info(f"Rerank: Reranking {len(combined_results)} documents...") # noqa: G004 try: rerank_service = get_rerank_service() @@ -808,22 +808,22 @@ async def agentic_retrieval( rerank_service=rerank_service, ) - logger.info(f"Rerank: Final Top {len(final_results)} selected") + logger.info(f"Rerank: Final Top {len(final_results)} selected") # noqa: G004 - except Exception as e: - logger.error(f"Final rerank failed: {e}, using top {config.final_top_n}") + except Exception as e: # noqa: BLE001 + logger.error(f"Final rerank failed: {e}, using top {config.final_top_n}") # noqa: G004 final_results = combined_results[: config.final_top_n] else: # No Reranker, directly return Top N final_results = combined_results[: config.final_top_n] - logger.info(f"No Rerank: Returning Top {len(final_results)}") + logger.info(f"No Rerank: Returning Top {len(final_results)}") # noqa: G004 metadata["final_count"] = len(final_results) metadata["total_latency_ms"] = (time.time() - start_time) * 1000 logger.info( - f"Complete: Final {len(final_results)} docs | Latency {metadata['total_latency_ms']:.0f}ms" + f"Complete: Final {len(final_results)} docs | Latency {metadata['total_latency_ms']:.0f}ms" # noqa: G004 ) - logger.info(f"{'=' * 60}\n") + logger.info(f"{'=' * 60}\n") # noqa: G004 return final_results, metadata diff --git a/methods/EverCore/src/agentic_layer/search_mem_service.py b/methods/EverCore/src/agentic_layer/search_mem_service.py index c73f3764..48fe29c1 100644 --- a/methods/EverCore/src/agentic_layer/search_mem_service.py +++ b/methods/EverCore/src/agentic_layer/search_mem_service.py @@ -965,7 +965,7 @@ async def _search_raw_message( return pending_messages - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to search raw messages: sender_id=%s, group_ids=%s, error=%s", sender_id, @@ -1076,7 +1076,7 @@ async def _search_episodic_memory_hierarchical( """Hierarchical retrieval for episodic_memory via DI-registered service, fallback to builtin hybrid.""" try: handler = get_bean("search_method_mrag") - except Exception: + except Exception: # noqa: BLE001 logger.warning( "Hierarchical search service not available, fallback to builtin hybrid" ) @@ -1147,7 +1147,7 @@ async def _search_extended_method( """ try: handler = get_bean(f"search_method_{method}") - except Exception: + except Exception: # noqa: BLE001 raise ValueError(f"Unknown search method: {method}") query_vector = await self._get_query_vector(query, retrieve_method=method) @@ -1622,7 +1622,7 @@ async def _verify_skill_relevance( return scored - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Skill relevance verification failed, returning all results: %s", e ) diff --git a/methods/EverCore/src/agentic_layer/vectorize_base.py b/methods/EverCore/src/agentic_layer/vectorize_base.py index dd54ebe8..0cd5d8fb 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_base.py +++ b/methods/EverCore/src/agentic_layer/vectorize_base.py @@ -39,7 +39,7 @@ def __init__(self, config): api_key, base_url, model = self._get_config_params() logger.info( - f"Initialized {self.__class__.__name__} | model={model} | base_url={base_url}" + f"Initialized {self.__class__.__name__} | model={model} | base_url={base_url}" # noqa: G004 ) @abstractmethod @@ -130,21 +130,21 @@ async def _make_request( collector.add( self.config.model, prompt_tokens, 0, call_type="embedding" ) - except Exception: + except Exception: # noqa: BLE001 pass return response - except Exception as e: + except Exception as e: # noqa: BLE001 error_msg = str(e) logger.error( - f"{self.__class__.__name__} API error (attempt {attempt + 1}/{self.config.max_retries}): {error_msg}" + f"{self.__class__.__name__} API error (attempt {attempt + 1}/{self.config.max_retries}): {error_msg}" # noqa: G004 ) # Log detailed error for debugging if "Connection" in error_msg or "timeout" in error_msg.lower(): logger.warning( - f"Network issue connecting to {self.config.base_url}: {error_msg}" + f"Network issue connecting to {self.config.base_url}: {error_msg}" # noqa: G004 ) if attempt < self.config.max_retries - 1: @@ -172,7 +172,7 @@ def _parse_embeddings_response(self, response) -> List[np.ndarray]: and len(emb) > self.config.dimensions ): logger.debug( - f"Client-side truncation: {len(emb)}D → {self.config.dimensions}D" + f"Client-side truncation: {len(emb)}D → {self.config.dimensions}D" # noqa: G004 ) emb = emb[: self.config.dimensions] @@ -241,7 +241,7 @@ async def get_embeddings_batch( embeddings_batches = [] for i, result in enumerate(results): if isinstance(result, Exception): - logger.error(f"Error processing batch {i}: {result}") + logger.error(f"Error processing batch {i}: {result}") # noqa: G004 embeddings_batches.append([]) else: embeddings_batches.append(result) diff --git a/methods/EverCore/src/agentic_layer/vectorize_service.py b/methods/EverCore/src/agentic_layer/vectorize_service.py index d0a3fee3..842150f9 100644 --- a/methods/EverCore/src/agentic_layer/vectorize_service.py +++ b/methods/EverCore/src/agentic_layer/vectorize_service.py @@ -248,7 +248,7 @@ def __init__(self, config: Optional[HybridVectorizeConfig] = None): ) logger.info( - f"Initialized HybridVectorizeService | " + f"Initialized HybridVectorizeService | " # noqa: G004 f"primary={config.primary_provider} | " f"fallback={config.fallback_provider} | " f"fallback_enabled={config.enable_fallback} | " @@ -390,7 +390,7 @@ async def execute_with_fallback( self.config._primary_failure_count = 0 return result - except Exception as primary_error: + except Exception as primary_error: # noqa: BLE001 primary_duration = time.perf_counter() - start_time # Increment failure count @@ -407,7 +407,7 @@ async def execute_with_fallback( ) logger.warning( - f"Primary service ({self.config.primary_provider}) {operation_name} failed " + f"Primary service ({self.config.primary_provider}) {operation_name} failed " # noqa: G004 f"(count: {self.config._primary_failure_count}): {primary_error}" ) @@ -431,7 +431,7 @@ async def execute_with_fallback( if self.config._primary_failure_count >= self.config.max_primary_failures: fallback_reason = 'max_failures_exceeded' logger.warning( - f"⚠️ Primary service exceeded max failures ({self.config.max_primary_failures}), " + f"⚠️ Primary service exceeded max failures ({self.config.max_primary_failures}), " # noqa: G004 f"using {self.config.fallback_provider} fallback" ) @@ -446,7 +446,7 @@ async def execute_with_fallback( fallback_start = time.perf_counter() try: logger.info( - f"🔄 Falling back to {self.config.fallback_provider} for {operation_name}" + f"🔄 Falling back to {self.config.fallback_provider} for {operation_name}" # noqa: G004 ) result = await fallback_func() fallback_duration = time.perf_counter() - fallback_start @@ -462,7 +462,7 @@ async def execute_with_fallback( return result - except Exception as fallback_error: + except Exception as fallback_error: # noqa: BLE001 fallback_duration = time.perf_counter() - fallback_start # Record fallback error @@ -479,7 +479,7 @@ async def execute_with_fallback( batch_size=batch_size, ) - logger.error(f"❌ Fallback also failed: {fallback_error}") + logger.error(f"❌ Fallback also failed: {fallback_error}") # noqa: G004 raise VectorizeError( f"Both primary and fallback services failed. " f"Primary ({self.config.primary_provider}): {primary_error}, " diff --git a/methods/EverCore/src/api_specs/memory_models.py b/methods/EverCore/src/api_specs/memory_models.py index a45e4cf3..1d62a8a4 100644 --- a/methods/EverCore/src/api_specs/memory_models.py +++ b/methods/EverCore/src/api_specs/memory_models.py @@ -47,7 +47,7 @@ def from_string(cls, role_str: Optional[str]) -> Optional['MessageSenderRole']: if role.value == role_lower: return role return None - except Exception: + except Exception: # noqa: BLE001 return None @classmethod diff --git a/methods/EverCore/src/api_specs/memory_types.py b/methods/EverCore/src/api_specs/memory_types.py index 21a082dc..95b90b6a 100644 --- a/methods/EverCore/src/api_specs/memory_types.py +++ b/methods/EverCore/src/api_specs/memory_types.py @@ -53,13 +53,13 @@ def from_string(cls, type_str: Optional[str]) -> Optional['RawDataType']: from core.observation.logger import get_logger logger = get_logger(__name__) - logger.error(f"No matching RawDataType found: {type_str}, returning None") + logger.error(f"No matching RawDataType found: {type_str}, returning None") # noqa: G004 return None - except Exception as e: + except Exception as e: # noqa: BLE001 from core.observation.logger import get_logger logger = get_logger(__name__) - logger.warning(f"Failed to convert type field: {type_str}, error: {e}") + logger.warning(f"Failed to convert type field: {type_str}, error: {e}") # noqa: G004 return None @@ -246,7 +246,7 @@ def _format_timestamp(self) -> Optional[str]: return self.timestamp if self.timestamp else None try: return to_iso_format(self.timestamp) - except Exception: + except Exception: # noqa: BLE001 return str(self.timestamp) if self.timestamp else None def to_dict(self) -> Dict[str, Any]: diff --git a/methods/EverCore/src/api_specs/request_converter.py b/methods/EverCore/src/api_specs/request_converter.py index 127b1ff7..28ae3ff6 100644 --- a/methods/EverCore/src/api_specs/request_converter.py +++ b/methods/EverCore/src/api_specs/request_converter.py @@ -50,12 +50,12 @@ def convert_dict_to_retrieve_mem_request( # Handle retrieve_method, use default keyword if not provided retrieve_method_str = data.get("retrieve_method", RetrieveMethod.KEYWORD.value) - logger.debug(f"[DEBUG] retrieve_method_str from data: {retrieve_method_str!r}") + logger.debug(f"[DEBUG] retrieve_method_str from data: {retrieve_method_str!r}") # noqa: G004 # Convert string to RetrieveMethod enum try: retrieve_method = RetrieveMethod(retrieve_method_str) - logger.debug(f"[DEBUG] converted to: {retrieve_method}") + logger.debug(f"[DEBUG] converted to: {retrieve_method}") # noqa: G004 except ValueError: raise ValueError( f"Invalid retrieve_method: {retrieve_method_str}. " @@ -91,7 +91,7 @@ def convert_dict_to_retrieve_mem_request( try: memory_types.append(MemoryType(mt)) except ValueError: - logger.error(f"Invalid memory_type: {mt}, skipping") + logger.error(f"Invalid memory_type: {mt}, skipping") # noqa: G004 elif isinstance(mt, MemoryType): memory_types.append(mt) @@ -136,7 +136,7 @@ def convert_dict_to_retrieve_mem_request( end_time=data.get("end_time", None), radius=radius, # COSINE similarity threshold ) - except Exception as e: + except Exception as e: # noqa: BLE001 raise ValueError(f"RetrieveMemRequest conversion failed: {e}") diff --git a/methods/EverCore/src/biz_layer/mem_db_operations.py b/methods/EverCore/src/biz_layer/mem_db_operations.py index b6cb5985..9340e02d 100644 --- a/methods/EverCore/src/biz_layer/mem_db_operations.py +++ b/methods/EverCore/src/biz_layer/mem_db_operations.py @@ -81,8 +81,8 @@ def _convert_episode_memory_to_doc( if hasattr(episode_memory, 'timestamp') and episode_memory.timestamp: try: timestamp_dt = from_iso_format(episode_memory.timestamp) - except Exception as e: - logger.debug(f"Timestamp conversion failed, using current time: {e}") + except Exception as e: # noqa: BLE001 + logger.debug(f"Timestamp conversion failed, using current time: {e}") # noqa: G004 timestamp_dt = current_time participants = episode_memory.participants @@ -234,7 +234,7 @@ def _convert_agent_case_to_doc( timestamp_dt = memcell.timestamp elif isinstance(memcell.timestamp, str): timestamp_dt = from_iso_format(memcell.timestamp) - except Exception: + except Exception: # noqa: BLE001 timestamp_dt = current_time # Extract user_id from first role='user' message's sender_id @@ -284,7 +284,7 @@ async def _save_memcell_to_database( try: timestamp_dt = from_iso_format(memcell.timestamp) except (ValueError, TypeError) as e: - logger.debug(f"Timestamp conversion failed, using current time: {e}") + logger.debug(f"Timestamp conversion failed, using current time: {e}") # noqa: G004 # Convert data type enum doc_type = None @@ -309,7 +309,7 @@ async def _save_memcell_to_database( if result: memcell.event_id = str(result.event_id) logger.info( - f"[mem_db_operations] MemCell saved successfully: {memcell.event_id}" + f"[mem_db_operations] MemCell saved successfully: {memcell.event_id}" # noqa: G004 ) # Publish MemCellCreatedEvent try: @@ -320,20 +320,17 @@ async def _save_memcell_to_database( ) await publisher.publish(event) logger.debug( - f"[mem_db_operations] MemCellCreatedEvent published: {memcell.event_id}" + f"[mem_db_operations] MemCellCreatedEvent published: {memcell.event_id}" # noqa: G004 ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( - f"[mem_db_operations] Failed to publish MemCellCreatedEvent: {e}" + f"[mem_db_operations] Failed to publish MemCellCreatedEvent: {e}" # noqa: G004 ) else: - logger.info(f"[mem_db_operations] MemCell save failed: {memcell.event_id}") + logger.info(f"[mem_db_operations] MemCell save failed: {memcell.event_id}") # noqa: G004 - except Exception as e: - logger.error(f"MemCell save failed: {e}") - import traceback - - traceback.print_exc() + except Exception: + logger.exception("MemCell save failed") return memcell @@ -370,7 +367,7 @@ async def _update_status_for_continuing_conversation( ) if not existing_status: logger.info( - f"Existing status not found, creating new status record: group_id={request.group_id}" + f"Existing status not found, creating new status record: group_id={request.group_id}" # noqa: G004 ) # Create new status record latest_dt = from_iso_format(latest_time) @@ -386,12 +383,12 @@ async def _update_status_for_continuing_conversation( ) if result: logger.info( - f"New status created successfully: group_id={request.group_id}" + f"New status created successfully: group_id={request.group_id}" # noqa: G004 ) return True else: logger.warning( - f"Failed to create new status: group_id={request.group_id}" + f"Failed to create new status: group_id={request.group_id}" # noqa: G004 ) return False @@ -427,8 +424,8 @@ async def _update_status_for_continuing_conversation( logger.warning("Conversation continuation status update failed") return False - except Exception as e: - logger.error(f"Conversation continuation status update failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Conversation continuation status update failed: {e}") # noqa: G004 return False @@ -530,6 +527,6 @@ async def _update_status_after_memcell_extraction( logger.warning("Status update after MemCell extraction failed") return False - except Exception as e: - logger.error(f"Status update after MemCell extraction failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Status update after MemCell extraction failed: {e}") # noqa: G004 return False diff --git a/methods/EverCore/src/biz_layer/mem_memorize.py b/methods/EverCore/src/biz_layer/mem_memorize.py index de401048..64688da5 100644 --- a/methods/EverCore/src/biz_layer/mem_memorize.py +++ b/methods/EverCore/src/biz_layer/mem_memorize.py @@ -1,6 +1,5 @@ from dataclasses import dataclass import time -import traceback from core.observation.stage_timer import timed, timed_parallel from api_specs.memory_types import ScenarioType @@ -85,7 +84,7 @@ def _is_agent_case_quality_sufficient( score = agent_case.quality_score if score is None or score < config.skill_min_quality_score: logger.info( - f"[AgentSkill] Skipping skill extraction: quality_score={score} " + f"[AgentSkill] Skipping skill extraction: quality_score={score} " # noqa: G004 f"< threshold={config.skill_min_quality_score}" ) return False @@ -112,7 +111,7 @@ async def _trigger_clustering( agent_case: Extracted AgentCase (if agent conversation), used for skill extraction """ logger.info( - f"[Clustering] Start triggering clustering: group_id={group_id}, event_id={memcell.event_id}, scene={scene}" + f"[Clustering] Start triggering clustering: group_id={group_id}, event_id={memcell.event_id}, scene={scene}" # noqa: G004 ) try: @@ -130,7 +129,7 @@ async def _trigger_clustering( # Get MongoDB storage cluster_storage = get_bean_by_type(MemSceneRawRepository) logger.info( - f"[Clustering] MemSceneRawRepository retrieved successfully: {type(cluster_storage)}" + f"[Clustering] MemSceneRawRepository retrieved successfully: {type(cluster_storage)}" # noqa: G004 ) # Create ClusterManager (pure computation component) @@ -165,7 +164,7 @@ async def _trigger_clustering( if has_case and agent_case.task_intent else episode_text ) - logger.info(f"[Clustering] ClusterManager created (has_case={has_case})") + logger.info(f"[Clustering] ClusterManager created (has_case={has_case})") # noqa: G004 # Convert MemCell to dictionary format required for clustering memcell_dict = { @@ -178,7 +177,7 @@ async def _trigger_clustering( } logger.debug( - f"[Clustering] Start clustering execution: {memcell_dict['event_id']}" + f"[Clustering] Start clustering execution: {memcell_dict['event_id']}" # noqa: G004 ) from core.lock.redis_distributed_lock import distributed_lock @@ -209,7 +208,7 @@ async def _trigger_clustering( ) as acquired: if not acquired: logger.error( - f"[Clustering] Failed to acquire lock for group {group_id}, " + f"[Clustering] Failed to acquire lock for group {group_id}, " # noqa: G004 f"skipping memcell {memcell.event_id}" ) return @@ -220,7 +219,7 @@ async def _trigger_clustering( MemSceneState.from_dict(state_dict) if state_dict else MemSceneState() ) logger.info( - f"[Clustering] Loaded clustering state: {len(mem_scene_state.event_ids)} clustered events" + f"[Clustering] Loaded clustering state: {len(mem_scene_state.event_ids)} clustered events" # noqa: G004 ) cluster_id, mem_scene_state = await cluster_manager.cluster_memcell( @@ -232,11 +231,11 @@ async def _trigger_clustering( if cluster_id: logger.debug( - f"[Clustering] ✅ MemCell {memcell.event_id} -> Cluster {cluster_id} (group: {group_id})" + f"[Clustering] ✅ MemCell {memcell.event_id} -> Cluster {cluster_id} (group: {group_id})" # noqa: G004 ) else: logger.warning( - f"[Clustering] ⚠️ MemCell {memcell.event_id} clustering returned None (group: {group_id})" + f"[Clustering] ⚠️ MemCell {memcell.event_id} clustering returned None (group: {group_id})" # noqa: G004 ) # ===== Phase 2: Profile extraction (with interval-based throttling) ===== @@ -296,7 +295,7 @@ async def _trigger_clustering( target_cluster_ids.append(cluster_id) logger.info( - f"[Profile] Timestamp-based selection: last_profile_ts={last_profile_ts}, " + f"[Profile] Timestamp-based selection: last_profile_ts={last_profile_ts}, " # noqa: G004 f"target_clusters={target_cluster_ids}" ) @@ -310,7 +309,7 @@ async def _trigger_clustering( ) else: logger.debug( - f"[Profile] Skipping extraction: total_memcells={total_memcell_count}, " + f"[Profile] Skipping extraction: total_memcells={total_memcell_count}, " # noqa: G004 f"interval={config.profile_extraction_interval}" ) @@ -345,7 +344,7 @@ async def _trigger_clustering( ) as skill_acquired: if not skill_acquired: logger.error( - f"[AgentSkill] Failed to acquire lock for group {group_id}, " + f"[AgentSkill] Failed to acquire lock for group {group_id}, " # noqa: G004 f"cluster {cluster_id}, skipping memcell {memcell.event_id}" ) return @@ -358,8 +357,8 @@ async def _trigger_clustering( ) except Exception as e: - logger.error( - f"[Clustering] ❌ Triggering clustering failed: {e}", exc_info=True + logger.error( # noqa: G201 + f"[Clustering] ❌ Triggering clustering failed: {e}", exc_info=True # noqa: G004 ) raise @@ -402,13 +401,13 @@ async def _trigger_profile_extraction( ) if total_memcell_count < config.profile_min_memcells: logger.debug( - f"[Profile] Clusters {cluster_ids} have only {total_memcell_count} memcells " + f"[Profile] Clusters {cluster_ids} have only {total_memcell_count} memcells " # noqa: G004 f"(requires {config.profile_min_memcells}), skipping extraction" ) return logger.info( - f"[Profile] Start extracting Profile: clusters={cluster_ids}, memcells={total_memcell_count}" + f"[Profile] Start extracting Profile: clusters={cluster_ids}, memcells={total_memcell_count}" # noqa: G004 ) # Get Profile storage @@ -445,8 +444,8 @@ async def _trigger_profile_extraction( try: fetched = await memcell_repo.get_by_event_ids(list(target_event_ids)) all_memcells = list(fetched.values()) - except Exception as e: - logger.warning(f"[Profile] Failed to fetch cluster memcells: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"[Profile] Failed to fetch cluster memcells: {e}") # noqa: G004 # Append current memcell as the last one (new_memcell) all_memcells.append(memcell) @@ -463,7 +462,7 @@ async def _trigger_profile_extraction( user_id_list = list(all_participants) logger.info( - f"[Profile] Context: clusters={len(cluster_ids)}, " + f"[Profile] Context: clusters={len(cluster_ids)}, " # noqa: G004 f"memcells={len(all_memcells) - 1}, new=1, users={len(user_id_list)}" ) @@ -475,12 +474,12 @@ async def _trigger_profile_extraction( old_profiles_dict = await profile_repo.get_all_profiles(group_id=group_id) old_profiles = list(old_profiles_dict.values()) if old_profiles_dict else [] logger.info( - f"[Profile] Loaded {len(old_profiles)} existing profiles for group={group_id}" + f"[Profile] Loaded {len(old_profiles)} existing profiles for group={group_id}" # noqa: G004 ) if old_profiles: for uid, p in old_profiles_dict.items(): keys = list(p.keys()) if isinstance(p, dict) else dir(p) - logger.info(f"[Profile] Profile for {uid}: keys={keys[:8]}") + logger.info(f"[Profile] Profile for {uid}: keys={keys[:8]}") # noqa: G004 # Extract profiles profile_scene = ( @@ -513,14 +512,14 @@ async def _trigger_profile_extraction( await profile_repo.save_profile( user_id, profile_data, metadata=metadata ) - logger.info(f"[Profile] ✅ Saved: user={user_id}") - except Exception as e: - logger.warning(f"[Profile] Failed to save profile: {e}") + logger.info(f"[Profile] ✅ Saved: user={user_id}") # noqa: G004 + except Exception as e: # noqa: BLE001 + logger.warning(f"[Profile] Failed to save profile: {e}") # noqa: G004 - logger.info(f"[Profile] ✅ Completed: {len(new_profiles)} profiles") + logger.info(f"[Profile] ✅ Completed: {len(new_profiles)} profiles") # noqa: G004 except Exception as e: - logger.error(f"[Profile] ❌ Profile extraction failed: {e}", exc_info=True) + logger.error(f"[Profile] ❌ Profile extraction failed: {e}", exc_info=True) # noqa: G004, G201 # Advance last_updated_ts even on failure to prevent repeated re-selection # of the same clusters. The data is "skipped" — acceptable tradeoff vs. @@ -545,11 +544,11 @@ async def _trigger_profile_extraction( trigger_index=False, ) logger.info( - f"[Profile] Advanced last_updated_ts to {memcell_ts} for {len(user_id_list)} users despite failure" + f"[Profile] Advanced last_updated_ts to {memcell_ts} for {len(user_id_list)} users despite failure" # noqa: G004 ) - except Exception as ts_err: + except Exception as ts_err: # noqa: BLE001 logger.warning( - f"[Profile] Failed to advance last_updated_ts on failure: {ts_err}" + f"[Profile] Failed to advance last_updated_ts on failure: {ts_err}" # noqa: G004 ) @@ -611,7 +610,7 @@ async def _trigger_agent_skill_extraction( ) logger.info( - f"[AgentSkill] Incremental extraction: cluster={cluster_id}, " + f"[AgentSkill] Incremental extraction: cluster={cluster_id}, " # noqa: G004 f"new_experience=1, existing_skills={len(existing_skills)}" ) @@ -637,11 +636,11 @@ async def _trigger_agent_skill_extraction( if extraction_result.deleted_ids: logger.info( - f"[AgentSkill] Retired skills for cluster={cluster_id}: " + f"[AgentSkill] Retired skills for cluster={cluster_id}: " # noqa: G004 f"ids={extraction_result.deleted_ids}" ) logger.info( - f"[AgentSkill] Extraction result for cluster={cluster_id}: " + f"[AgentSkill] Extraction result for cluster={cluster_id}: " # noqa: G004 f"added={len(extraction_result.added_records)}, " f"updated={len(extraction_result.updated_records)}, " f"retired={len(extraction_result.deleted_ids)}" @@ -670,15 +669,15 @@ async def _trigger_agent_skill_extraction( inserted_count += 1 else: logger.warning( - f"[AgentSkill] Milvus skip (no vector): record={record.id}" + f"[AgentSkill] Milvus skip (no vector): record={record.id}" # noqa: G004 ) logger.info( - f"[AgentSkill] Milvus synced for cluster={cluster_id}: " + f"[AgentSkill] Milvus synced for cluster={cluster_id}: " # noqa: G004 f"inserted={inserted_count}, removed={len(remove_ids)}" ) - except Exception as milvus_exc: + except Exception as milvus_exc: # noqa: BLE001 logger.warning( - f"[AgentSkill] Milvus sync failed for cluster={cluster_id}: {milvus_exc}" + f"[AgentSkill] Milvus sync failed for cluster={cluster_id}: {milvus_exc}" # noqa: G004 ) # ES sync: delete stale entries -> insert new/updated @@ -690,17 +689,17 @@ async def _trigger_agent_skill_extraction( es_doc = AgentSkillConverter.from_mongo(record) await agent_skill_es_repo.create(es_doc) logger.info( - f"[AgentSkill] ES synced for cluster={cluster_id}: " + f"[AgentSkill] ES synced for cluster={cluster_id}: " # noqa: G004 f"inserted={len(upsert_records)}, removed={len(remove_ids)}" ) - except Exception as es_exc: + except Exception as es_exc: # noqa: BLE001 logger.warning( - f"[AgentSkill] ES sync failed for cluster={cluster_id}: {es_exc}" + f"[AgentSkill] ES sync failed for cluster={cluster_id}: {es_exc}" # noqa: G004 ) except Exception as e: - logger.error( - f"[AgentSkill] Skill extraction failed for cluster={cluster_id}: {e}", + logger.error( # noqa: G201 + f"[AgentSkill] Skill extraction failed for cluster={cluster_id}: {e}", # noqa: G004 exc_info=True, ) @@ -875,8 +874,8 @@ async def _clustering_with_metrics(): cluster_start = time.perf_counter() try: await _update_memcell_and_cluster(state) - except Exception as e: - logger.error(f"[MemCell Processing] ❌ Background clustering failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[MemCell Processing] ❌ Background clustering failed: {e}") # noqa: G004 finally: record_extraction_stage( space_id=space_id, @@ -937,7 +936,7 @@ async def _extract_episodes(state: ExtractionState, memory_manager: MemoryManage tasks = [_create_episode_task(state, memory_manager, None)] else: logger.info( - f"[MemCell Processing] team scene, extract group + {len(state.participants)} personal Episodes" + f"[MemCell Processing] team scene, extract group + {len(state.participants)} personal Episodes" # noqa: G004 ) tasks = [_create_episode_task(state, memory_manager, None)] tasks.extend( @@ -972,7 +971,7 @@ def _process_episode_results(state: ExtractionState, results: List[Any]): group_episode = results[0] if results else None if isinstance(group_episode, Exception): logger.error( - f"[MemCell Processing] ❌ Group Episode exception: {group_episode}" + f"[MemCell Processing] ❌ Group Episode exception: {group_episode}" # noqa: G004 ) group_episode = None elif group_episode: @@ -987,7 +986,7 @@ def _process_episode_results(state: ExtractionState, results: List[Any]): for user_id, result in zip(state.participants, results[1:], strict=False): if isinstance(result, Exception): logger.error( - f"[MemCell Processing] ❌ Personal Episode exception: user_id={user_id}" + f"[MemCell Processing] ❌ Personal Episode exception: user_id={user_id}" # noqa: G004 ) continue if result: @@ -995,7 +994,7 @@ def _process_episode_results(state: ExtractionState, results: List[Any]): result.parent_id = state.parent_id state.episode_memories.append(result) logger.info( - f"[MemCell Processing] ✅ Personal Episode successful: user_id={user_id}" + f"[MemCell Processing] ✅ Personal Episode successful: user_id={user_id}" # noqa: G004 ) @@ -1016,10 +1015,10 @@ async def _update_memcell_and_cluster(state: ExtractionState): agent_case=state.agent_case, ) logger.info( - f"[MemCell Processing] ✅ Clustering completed (scene={state.scene})" + f"[MemCell Processing] ✅ Clustering completed (scene={state.scene})" # noqa: G004 ) - except Exception as e: - logger.error(f"[MemCell Processing] ❌ Failed to trigger clustering: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[MemCell Processing] ❌ Failed to trigger clustering: {e}") # noqa: G004 async def _process_memories(state: ExtractionState) -> int: @@ -1156,11 +1155,11 @@ async def _save_agent_case(state: ExtractionState) -> int: payloads = [MemoryDocPayload(MemoryType.AGENT_CASE, doc)] await save_memory_docs(payloads) logger.info( - f"[MemCell Processing] AgentCase saved: intent='{agent_case.task_intent[:80]}'" + f"[MemCell Processing] AgentCase saved: intent='{agent_case.task_intent[:80]}'" # noqa: G004 ) return 1 - except Exception as e: - logger.error(f"[MemCell Processing] Failed to save AgentCase: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[MemCell Processing] Failed to save AgentCase: {e}") # noqa: G004 return 0 @@ -1172,7 +1171,7 @@ def _clone_episodes_for_users(state: ExtractionState) -> List[EpisodeMemory]: group_ep = state.group_episode_memories[0] for user_id in state.participants: cloned.append(replace(group_ep, user_id=user_id, user_name=user_id)) - logger.info(f"[MemCell Processing] Copied group Episode to {len(cloned)} users") + logger.info(f"[MemCell Processing] Copied group Episode to {len(cloned)} users") # noqa: G004 return cloned @@ -1254,7 +1253,7 @@ async def _save_foresight_and_atomic_fact( ] ) logger.info( - f"[MemCell Processing] Copied Foresight/AtomicFact to {len(user_ids)} users" + f"[MemCell Processing] Copied Foresight/AtomicFact to {len(user_ids)} users" # noqa: G004 ) payloads = [] @@ -1312,8 +1311,8 @@ async def _foresight_and_atomic_facts_with_metrics( state, foresight_memories, atomic_facts ) except Exception as e: - logger.error( - f"[ForesightAF] ❌ Background extraction/save failed: {e}", exc_info=True + logger.error( # noqa: G201 + f"[ForesightAF] ❌ Background extraction/save failed: {e}", exc_info=True # noqa: G004 ) finally: record_extraction_stage( @@ -1340,7 +1339,7 @@ async def preprocess_conv_request( 5. Boundary detection handled by subsequent logic (will clear or retain after detection) """ - logger.info(f"[preprocess] Start processing: group_id={request.group_id}") + logger.info(f"[preprocess] Start processing: group_id={request.group_id}") # noqa: G004 # Check if there is new data if not request.new_raw_data_list: @@ -1377,12 +1376,11 @@ async def preprocess_conv_request( request.history_raw_data_list = accumulated # new_raw_data_list stays empty; extractor handles flush+empty-new case logger.info( - f"[preprocess] Flush: loaded {len(accumulated)} accumulated messages as history" + f"[preprocess] Flush: loaded {len(accumulated)} accumulated messages as history" # noqa: G004 ) return request - except Exception as e: - logger.error(f"[preprocess] Flush data read failed: {e}") - traceback.print_exc() + except Exception: + logger.exception("[preprocess] Flush data read failed") return None # Use conversation_data_repo for read-then-store operation @@ -1401,7 +1399,7 @@ async def preprocess_conv_request( if status and status.last_memcell_time: start_time = status.last_memcell_time logger.info( - f"[preprocess] Using last_memcell_time as start_time: {start_time}" + f"[preprocess] Using last_memcell_time as start_time: {start_time}" # noqa: G004 ) # Step 1: Get historical messages, excluding current request's messages @@ -1416,7 +1414,7 @@ async def preprocess_conv_request( ) logger.info( - f"[preprocess] Read {len(history_raw_data_list)} historical messages (excluded {len(new_message_ids)} new, start_time={start_time})" + f"[preprocess] Read {len(history_raw_data_list)} historical messages (excluded {len(new_message_ids)} new, start_time={start_time})" # noqa: G004 ) # Update request @@ -1424,14 +1422,13 @@ async def preprocess_conv_request( # new_raw_data_list remains unchanged (the newly passed messages) logger.info( - f"[preprocess] Completed: {len(history_raw_data_list)} historical, {len(request.new_raw_data_list)} new messages" + f"[preprocess] Completed: {len(history_raw_data_list)} historical, {len(request.new_raw_data_list)} new messages" # noqa: G004 ) return request - except Exception as e: - logger.error(f"[preprocess] Data read failed: {e}") - traceback.print_exc() + except Exception: + logger.exception("[preprocess] Data read failed") # Use original request if read fails return request @@ -1475,8 +1472,8 @@ async def update_status_when_no_memcell( status_repo, request, latest_time, current_time ) - except Exception as e: - logger.error(f"Failed to update status table: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to update status table: {e}") # noqa: G004 else: pass @@ -1508,8 +1505,8 @@ async def update_status_after_memcell( "[mem_memorize] Memory extraction completed, status table updated" ) - except Exception as e: - logger.error(f"Final status table update failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Final status table update failed: {e}") # noqa: G004 else: pass @@ -1612,8 +1609,8 @@ async def save_memory_docs( agent_case_es_repo = get_bean_by_type(AgentCaseEsRepository) es_doc = AgentCaseConverter.from_mongo(saved_doc) await agent_case_es_repo.create(es_doc) - except Exception as es_exc: - logger.warning(f"[mem_memorize] AgentCase ES sync failed: {es_exc}") + except Exception as es_exc: # noqa: BLE001 + logger.warning(f"[mem_memorize] AgentCase ES sync failed: {es_exc}") # noqa: G004 # Milvus sync try: @@ -1630,9 +1627,9 @@ async def save_memory_docs( logger.warning( "[mem_memorize] Skipping AgentCase Milvus write: vector empty or missing" ) - except Exception as milvus_exc: + except Exception as milvus_exc: # noqa: BLE001 logger.warning( - f"[mem_memorize] AgentCase Milvus sync failed: {milvus_exc}" + f"[mem_memorize] AgentCase Milvus sync failed: {milvus_exc}" # noqa: G004 ) saved_result[MemoryType.AGENT_CASE] = saved_agent_cases @@ -1687,14 +1684,14 @@ async def memorize(request: MemorizeRequest) -> int: Returns: int: Number of memories extracted (0 if no boundary detected or extraction failed) """ - logger.info(f"[mem_memorize] request.current_time: {request.current_time}") + logger.info(f"[mem_memorize] request.current_time: {request.current_time}") # noqa: G004 # Get current time if request.current_time: current_time = request.current_time else: current_time = get_now_with_timezone() + timedelta(seconds=1) - logger.info(f"[mem_memorize] Current time: {current_time}") + logger.info(f"[mem_memorize] Current time: {current_time}") # noqa: G004 conversation_data_repo = get_bean_by_type(ConversationDataRepository) @@ -1718,7 +1715,7 @@ async def memorize(request: MemorizeRequest) -> int: llm_custom_setting = await _load_llm_custom_setting() if llm_custom_setting: logger.info( - f"[mem_memorize] Using llm_custom_setting from global config for group {request.group_id}" + f"[mem_memorize] Using llm_custom_setting from global config for group {request.group_id}" # noqa: G004 ) # Boundary detection @@ -1727,12 +1724,12 @@ async def memorize(request: MemorizeRequest) -> int: raw_data_type = request.raw_data_type.value if request.raw_data_type else 'unknown' logger.info("=" * 80) - logger.info(f"[Boundary Detection] Start detection: group_id={request.group_id}") + logger.info(f"[Boundary Detection] Start detection: group_id={request.group_id}") # noqa: G004 logger.info( - f"[Boundary Detection] Temporary stored historical messages: {len(request.history_raw_data_list)} messages" + f"[Boundary Detection] Temporary stored historical messages: {len(request.history_raw_data_list)} messages" # noqa: G004 ) logger.info( - f"[Boundary Detection] New messages: {len(request.new_raw_data_list)} messages" + f"[Boundary Detection] New messages: {len(request.new_raw_data_list)} messages" # noqa: G004 ) logger.info("=" * 80) @@ -1756,7 +1753,7 @@ async def memorize(request: MemorizeRequest) -> int: duration_seconds=time.perf_counter() - memcell_start, ) logger.debug( - f"[mem_memorize] Extracting MemCell took: {time.perf_counter() - memcell_start} seconds" + f"[mem_memorize] Extracting MemCell took: {time.perf_counter() - memcell_start} seconds" # noqa: G004 ) memcells, status_result = memcell_result @@ -1764,7 +1761,7 @@ async def memorize(request: MemorizeRequest) -> int: # Check boundary detection result logger.info("=" * 80) logger.info( - f"[Boundary Detection Result] memcells={len(memcells)}, " + f"[Boundary Detection Result] memcells={len(memcells)}, " # noqa: G004 f"should_wait={status_result.should_wait}" ) logger.info("=" * 80) @@ -1778,7 +1775,7 @@ async def memorize(request: MemorizeRequest) -> int: session_id=request.session_id, ) logger.info( - f"[mem_memorize] No boundary, confirmed {len(request.new_raw_data_list)} messages to accumulation" + f"[mem_memorize] No boundary, confirmed {len(request.new_raw_data_list)} messages to accumulation" # noqa: G004 ) await update_status_when_no_memcell( request, status_result, current_time, request.raw_data_type @@ -1800,12 +1797,12 @@ async def memorize(request: MemorizeRequest) -> int: ) if delete_success: logger.debug( - f"[mem_memorize] Flush mode: all messages marked as used, " + f"[mem_memorize] Flush mode: all messages marked as used, " # noqa: G004 f"group_id={request.group_id}" ) else: logger.warning( - f"[mem_memorize] Failed to clear conversation history: group_id={request.group_id}" + f"[mem_memorize] Failed to clear conversation history: group_id={request.group_id}" # noqa: G004 ) else: # Non-flush: consumed messages marked as used, remaining start next window @@ -1816,22 +1813,21 @@ async def memorize(request: MemorizeRequest) -> int: ) if delete_success: logger.debug( - f"[mem_memorize] Consumed messages marked as used " + f"[mem_memorize] Consumed messages marked as used " # noqa: G004 f"(remaining={len(remaining_raw_data)}): group_id={request.group_id}" ) else: logger.warning( - f"[mem_memorize] Failed to mark consumed messages: group_id={request.group_id}" + f"[mem_memorize] Failed to mark consumed messages: group_id={request.group_id}" # noqa: G004 ) if remaining_raw_data: await conversation_data_repo.save_conversation_data( remaining_raw_data, request.group_id, session_id=request.session_id ) - except Exception as e: - logger.error( - f"[mem_memorize] Exception while marking conversation history: {e}" + except Exception: + logger.exception( + "[mem_memorize] Exception while marking conversation history" ) - traceback.print_exc() # Save and process all extracted MemCells memories_count = 0 @@ -1842,7 +1838,7 @@ async def memorize(request: MemorizeRequest) -> int: memcell = await _save_memcell_to_database( memcell, current_time, session_id=request.session_id ) - logger.info(f"[mem_memorize] Saved MemCell: {memcell.event_id}") + logger.info(f"[mem_memorize] Saved MemCell: {memcell.event_id}") # noqa: G004 with timed("process_memory_extraction"): count = await process_memory_extraction( memcell, request, memory_manager, current_time @@ -1850,11 +1846,10 @@ async def memorize(request: MemorizeRequest) -> int: memories_count += count logger.info( - f"[mem_memorize] ✅ Memory extraction completed, " + f"[mem_memorize] ✅ Memory extraction completed, " # noqa: G004 f"memcells={len(memcells)}, total_memories={memories_count}" ) return memories_count - except Exception as e: - logger.error(f"[mem_memorize] ❌ Memory extraction failed: {e}") - traceback.print_exc() + except Exception: + logger.exception("[mem_memorize] ❌ Memory extraction failed") return 0 diff --git a/methods/EverCore/src/biz_layer/mem_sync.py b/methods/EverCore/src/biz_layer/mem_sync.py index de819f9e..c661120b 100644 --- a/methods/EverCore/src/biz_layer/mem_sync.py +++ b/methods/EverCore/src/biz_layer/mem_sync.py @@ -114,7 +114,7 @@ async def sync_foresight( # Read embedding from MongoDB, skip if not exists if not foresight.vector: logger.warning( - f"Foresight {foresight.id} has no embedding, skipping sync" + f"Foresight {foresight.id} has no embedding, skipping sync" # noqa: G004 ) return stats @@ -124,7 +124,7 @@ async def sync_foresight( milvus_entity = ForesightMilvusConverter.from_mongo(foresight) await self.foresight_milvus_repo.insert(milvus_entity, flush=False) stats["foresight"] += 1 - logger.debug(f"Foresight synced to Milvus: {foresight.id}") + logger.debug(f"Foresight synced to Milvus: {foresight.id}") # noqa: G004 # Sync to ES if sync_to_es: @@ -132,10 +132,10 @@ async def sync_foresight( es_doc = ForesightConverter.from_mongo(foresight) await self.foresight_es_repo.create(es_doc) stats["es_records"] += 1 - logger.debug(f"Foresight synced to ES: {foresight.id}") + logger.debug(f"Foresight synced to ES: {foresight.id}") # noqa: G004 except Exception as e: - logger.error(f"Failed to sync foresight: {e}", exc_info=True) + logger.error(f"Failed to sync foresight: {e}", exc_info=True) # noqa: G004, G201 raise return stats @@ -162,7 +162,7 @@ async def sync_atomic_fact( # Read existing vector from MongoDB if not atomic_fact_record.vector: logger.warning( - f"Atomic fact {atomic_fact_record.id} has no embedding, skipping sync" + f"Atomic fact {atomic_fact_record.id} has no embedding, skipping sync" # noqa: G004 ) return stats @@ -172,7 +172,7 @@ async def sync_atomic_fact( milvus_entity = AtomicFactMilvusConverter.from_mongo(atomic_fact_record) await self.atomic_fact_milvus_repo.insert(milvus_entity, flush=False) stats["atomic_fact"] += 1 - logger.debug(f"Atomic fact synced to Milvus: {atomic_fact_record.id}") + logger.debug(f"Atomic fact synced to Milvus: {atomic_fact_record.id}") # noqa: G004 # Sync to ES if sync_to_es: @@ -180,10 +180,10 @@ async def sync_atomic_fact( es_doc = AtomicFactConverter.from_mongo(atomic_fact_record) await self.atomic_fact_es_repo.create(es_doc) stats["es_records"] += 1 - logger.debug(f"Atomic fact synced to ES: {atomic_fact_record.id}") + logger.debug(f"Atomic fact synced to ES: {atomic_fact_record.id}") # noqa: G004 except Exception as e: - logger.error(f"Failed to sync atomic fact: {e}", exc_info=True) + logger.error(f"Failed to sync atomic fact: {e}", exc_info=True) # noqa: G004, G201 raise return stats @@ -214,14 +214,14 @@ async def sync_batch_foresights( total_stats["foresight"] += stats.get("foresight", 0) total_stats["es_records"] += stats.get("es_records", 0) except Exception as e: - logger.error( - f"Failed to batch sync foresight: {foresight_mem.id}, error: {e}", + logger.error( # noqa: G201 + f"Failed to batch sync foresight: {foresight_mem.id}, error: {e}", # noqa: G004 exc_info=True, ) # Do not silently swallow exceptions logger.info( - f"✅ Foresight Milvus flush completed: {total_stats['foresight']} records" + f"✅ Foresight Milvus flush completed: {total_stats['foresight']} records" # noqa: G004 ) return total_stats @@ -252,15 +252,15 @@ async def sync_batch_atomic_facts( total_stats["atomic_fact"] += stats.get("atomic_fact", 0) total_stats["es_records"] += stats.get("es_records", 0) except Exception as e: - logger.error( - f"Failed to batch sync atomic fact: {fact_record.id}, error: {e}", + logger.error( # noqa: G201 + f"Failed to batch sync atomic fact: {fact_record.id}, error: {e}", # noqa: G004 exc_info=True, ) # Do not silently swallow exceptions, let it surface raise logger.info( - f"Atomic fact Milvus flush completed: {total_stats['atomic_fact']} records" + f"Atomic fact Milvus flush completed: {total_stats['atomic_fact']} records" # noqa: G004 ) return total_stats diff --git a/methods/EverCore/src/bootstrap.py b/methods/EverCore/src/bootstrap.py index 6f510a1a..ab3cb8b6 100644 --- a/methods/EverCore/src/bootstrap.py +++ b/methods/EverCore/src/bootstrap.py @@ -121,8 +121,8 @@ async def setup_project_context(env_file=".env", mock_mode=False): logger.info("✅ Application lifespan started successfully") else: logger.warning("⚠️ app instance has no start_lifespan method") - except Exception as e: - logger.warning(f"⚠️ Error starting application lifespan: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"⚠️ Error starting application lifespan: {e}") # noqa: G004 # Do not raise exception, continue execution @@ -222,9 +222,7 @@ async def async_main(): file=sys.stderr, ) print(f"Original error: {e}", file=sys.stderr) - import traceback - - traceback.print_exc() + logger.exception("Module mode execution failed") sys.exit(1) else: # For other import errors, raise directly @@ -238,11 +236,8 @@ async def async_main(): raise # Re-raise to propagate the exit code else: print("\n📋 Script execution completed successfully") - except Exception as e: - print(f"\n❌ Script execution error: {e}", file=sys.stderr) - import traceback - - traceback.print_exc() + except Exception: + logger.exception("Script execution error") finally: # Restore original sys.argv sys.argv = original_argv @@ -256,11 +251,9 @@ def main(): except KeyboardInterrupt: print("\n⚠️ User interrupted execution") sys.exit(1) - except Exception as e: - print(f"\n❌ Execution failed: {e}", file=sys.stderr) - import traceback - - traceback.print_exc() + except Exception: + logger.exception("Execution failed") + sys.exit(1) if __name__ == "__main__": diff --git a/methods/EverCore/src/common_utils/cli_ui.py b/methods/EverCore/src/common_utils/cli_ui.py index 0cb21ce7..e6bfbde3 100644 --- a/methods/EverCore/src/common_utils/cli_ui.py +++ b/methods/EverCore/src/common_utils/cli_ui.py @@ -259,7 +259,7 @@ def get_terminal_width(fallback: int = 80, min_width: int = 40) -> int: """Return the current terminal column width with reasonable bounds.""" try: width = shutil.get_terminal_size((fallback, 24)).columns - except Exception: + except Exception: # noqa: BLE001 width = fallback if width < min_width: width = min_width diff --git a/methods/EverCore/src/common_utils/datetime_utils.py b/methods/EverCore/src/common_utils/datetime_utils.py index 0fae1855..466b15eb 100644 --- a/methods/EverCore/src/common_utils/datetime_utils.py +++ b/methods/EverCore/src/common_utils/datetime_utils.py @@ -143,7 +143,7 @@ def to_timestamp_ms_universal(time_value) -> int: return to_timestamp_ms_universal(str(time_value)) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "[DateTimeUtils] to_timestamp_ms_universal - Error converting %s: %s", time_value, @@ -245,7 +245,7 @@ def from_iso_format( # Lenient mode: return current time on failure try: return _parse_datetime_core(create_time, target_timezone) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "[DateTimeUtils] from_iso_format - Error converting time: %s", str(e) ) diff --git a/methods/EverCore/src/common_utils/url_extractor.py b/methods/EverCore/src/common_utils/url_extractor.py index f3aaadc7..5be8bcef 100644 --- a/methods/EverCore/src/common_utils/url_extractor.py +++ b/methods/EverCore/src/common_utils/url_extractor.py @@ -72,7 +72,7 @@ async def extract_metadata( return metadata - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to extract URL metadata: %s, error: %s", url, str(e)) return self._create_error_metadata(url, str(e)) @@ -105,7 +105,7 @@ async def _get_final_url(self, url: str) -> str: async with session.head(url, allow_redirects=True) as response: return str(response.url) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("Failed to get final URL: %s, error: %s", url, str(e)) return url @@ -169,7 +169,7 @@ async def _fetch_html_content(self, url: str) -> Optional[str]: return content - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to get HTML content: %s, error: %s", url, str(e)) return None @@ -237,7 +237,7 @@ def _extract_metadata_from_soup( # Clean and validate data metadata = self._clean_metadata(metadata) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to parse metadata: %s, error: %s", url, str(e)) return metadata @@ -469,7 +469,7 @@ def _is_valid_url(self, url: str) -> bool: try: result = urlparse(url) return all([result.scheme, result.netloc]) - except Exception: + except Exception: # noqa: BLE001 return False def _create_empty_metadata( diff --git a/methods/EverCore/src/core/addons/addonize/addon_bean_order_strategy.py b/methods/EverCore/src/core/addons/addonize/addon_bean_order_strategy.py index aca0b872..2a98f266 100644 --- a/methods/EverCore/src/core/addons/addonize/addon_bean_order_strategy.py +++ b/methods/EverCore/src/core/addons/addonize/addon_bean_order_strategy.py @@ -190,7 +190,7 @@ def _replace_strategy(): " 📌 Addon priority configuration: %s (environment variable: ADDON_PRIORITY)", AddonBeanOrderStrategy.load_addon_priority_map(), ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to replace Bean ordering strategy: %s", e) diff --git a/methods/EverCore/src/core/addons/addons_registry.py b/methods/EverCore/src/core/addons/addons_registry.py index 7af5c7e3..35137448 100644 --- a/methods/EverCore/src/core/addons/addons_registry.py +++ b/methods/EverCore/src/core/addons/addons_registry.py @@ -204,7 +204,7 @@ def load_entrypoints(self) -> 'AddonsRegistry': ep.load() logger.info(" ✅ Loaded entrypoint: %s", ep.name) - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # noqa: BLE001 logger.error(" ❌ Failed to load entrypoint %s: %s", ep.name, e) logger.info( @@ -215,7 +215,7 @@ def load_entrypoints(self) -> 'AddonsRegistry': logger.warning( "⚠️ importlib.metadata is not available, skipping entry points loading" ) - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # noqa: BLE001 logger.error("❌ Failed to load addons entry points: %s", e) return self diff --git a/methods/EverCore/src/core/asynctasks/task_manager.py b/methods/EverCore/src/core/asynctasks/task_manager.py index 4626f18b..a172cfe3 100644 --- a/methods/EverCore/src/core/asynctasks/task_manager.py +++ b/methods/EverCore/src/core/asynctasks/task_manager.py @@ -17,6 +17,7 @@ from core.context.context_manager import ContextManager from core.context.context import get_current_user_info from core.di.decorators import component +from core.longjob.interfaces import RetryConfig from core.observation.logger import get_logger from core.authorize.enums import Role @@ -50,16 +51,6 @@ class TaskResult: metadata: Dict[str, Any] = field(default_factory=dict) -@dataclass -class RetryConfig: - """Retry configuration""" - - max_retries: int = 1 - retry_delay: float = 1.0 # seconds - exponential_backoff: bool = True - max_retry_delay: float = 60.0 # seconds - - @dataclass class TaskFunction: """Task function""" @@ -159,7 +150,7 @@ def register_task(self, task_function: TaskFunction) -> None: task_function: Task function to register """ self._task_registry[task_function.name] = task_function - logger.info(f"Task registered: {task_function.name}") + logger.info(f"Task registered: {task_function.name}") # noqa: G004 def scan_and_register_tasks(self, registry: TaskScanDirectoriesRegistry) -> None: """ @@ -185,7 +176,7 @@ def _scan_directory_for_tasks(self, directory: str) -> None: relative_path = Path(directory).resolve().relative_to(src_dir) package_name = ".".join(relative_path.parts) - logger.info(f"Scanning task package: {package_name}") + logger.info(f"Scanning task package: {package_name}") # noqa: G004 # Import package and scan try: @@ -200,19 +191,19 @@ def _scan_directory_for_tasks(self, directory: str) -> None: try: module = importlib.import_module(module_name) self._scan_module_for_tasks(module) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"Failed to import module: {module_name}, error: {e}" + f"Failed to import module: {module_name}, error: {e}" # noqa: G004 ) else: # This is a module, scan directly self._scan_module_for_tasks(package) - except Exception as e: - logger.error(f"Failed to import package: {package_name}, error: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to import package: {package_name}, error: {e}") # noqa: G004 - except Exception as e: - logger.error(f"Failed to scan directory: {directory}, error: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to scan directory: {directory}, error: {e}") # noqa: G004 def _scan_module_for_tasks(self, module: Any) -> None: """ @@ -235,16 +226,16 @@ def _scan_module_for_tasks(self, module: Any) -> None: if isinstance(attr, TaskFunction): self.register_task(attr) logger.info( - f"Task found in module {module.__name__}: {attr.name}" + f"Task found in module {module.__name__}: {attr.name}" # noqa: G004 ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.debug( - f"Failed to get module attribute: {module.__name__}.{attr_name}, error: {e}" + f"Failed to get module attribute: {module.__name__}.{attr_name}, error: {e}" # noqa: G004 ) - except Exception as e: - logger.error(f"Failed to scan module tasks: {module.__name__}, error: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to scan module tasks: {module.__name__}, error: {e}") # noqa: G004 async def enqueue_task( self, @@ -342,7 +333,7 @@ async def enqueue_task( user_id_for_log = user_data.get("user_id") if user_data else "unknown" logger.info( - f"Task added to queue: {task_id}, task name: {task_name}, user: {user_id_for_log}" + f"Task added to queue: {task_id}, task name: {task_name}, user: {user_id_for_log}" # noqa: G004 ) return task_id @@ -377,7 +368,7 @@ async def execute_task_with_context( from core.context.context import set_current_app_info set_current_app_info(app_info) - logger.debug(f"🔧 app_info_context restored: {app_info}") + logger.debug(f"🔧 app_info_context restored: {app_info}") # noqa: G004 # Use ContextManager to execute task, automatically injecting user context and database session # 🔧 Configurable session isolation: Only force new session when explicitly needed @@ -394,7 +385,7 @@ async def execute_task_with_context( task_id = task_context.get("task_id") user_id = user_data.get("user_id") if user_data else "unknown" logger.info( - f"Task execution completed (independent session): {task_id}, user: {user_id}" + f"Task execution completed (independent session): {task_id}, user: {user_id}" # noqa: G004 ) return result @@ -440,8 +431,8 @@ async def get_task_result(self, task_id: str) -> Optional[TaskResult]: return result - except Exception as e: - logger.error(f"Failed to get task result: {task_id}, error: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to get task result: {task_id}, error: {str(e)}") # noqa: G004 return None def _map_arq_status_to_task_status(self, arq_status: str) -> TaskStatus: @@ -479,10 +470,10 @@ async def cancel_task(self, task_id: str) -> bool: try: job = Job(task_id, pool) await job.abort() - logger.info(f"Task cancelled: {task_id}") + logger.info(f"Task cancelled: {task_id}") # noqa: G004 return True - except Exception as e: - logger.error(f"Failed to cancel task: {task_id}, error: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to cancel task: {task_id}, error: {str(e)}") # noqa: G004 return False async def delete_task(self, task_id: str) -> bool: @@ -500,10 +491,10 @@ async def delete_task(self, task_id: str) -> bool: try: # Delete task record await pool.delete(f"arq:job:{task_id}") - logger.info(f"Task deleted: {task_id}") + logger.info(f"Task deleted: {task_id}") # noqa: G004 return True - except Exception as e: - logger.error(f"Failed to delete task: {task_id}, error: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to delete task: {task_id}, error: {str(e)}") # noqa: G004 return False async def list_tasks( @@ -551,8 +542,8 @@ async def list_tasks( return tasks - except Exception as e: - logger.error(f"Failed to list tasks: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to list tasks: {str(e)}") # noqa: G004 return [] async def get_task_count(self, status: Optional[TaskStatus] = None) -> int: diff --git a/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py b/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py index 85a1b47c..8d7c01b7 100644 --- a/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py +++ b/methods/EverCore/src/core/cache/redis_cache_queue/redis_data_processor.py @@ -90,7 +90,7 @@ def deserialize_data(data: Union[str, bytes]) -> Any: result = pickle.loads(pickle_data) logger.debug("Pickle deserialization succeeded") return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Pickle deserialization failed: %s", str(e)) return data else: @@ -225,7 +225,7 @@ def process_data_from_storage(member: Union[str, bytes]) -> Dict[str, Any]: try: parsed_data = RedisDataProcessor.deserialize_data(raw_data) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to deserialize data: member=%s, error=%s", ( diff --git a/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py b/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py index 87791647..314bbaed 100644 --- a/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py +++ b/methods/EverCore/src/core/cache/redis_cache_queue/redis_length_cache_manager.py @@ -480,7 +480,7 @@ async def get_by_timestamp_range( } ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to parse message: i=%d, message=%s, error=%s", i, @@ -502,7 +502,7 @@ async def get_by_timestamp_range( return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to retrieve data by timestamp range: key=%s, error=%s", key, diff --git a/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py b/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py index 73a13fe4..b15bf60f 100644 --- a/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py +++ b/methods/EverCore/src/core/cache/redis_cache_queue/redis_windows_cache_manager.py @@ -277,7 +277,7 @@ async def append( ) return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis data append failed: key=%s, error=%s", key, str(e)) return False @@ -295,7 +295,7 @@ async def get_queue_size(self, key: str) -> int: client = await self.redis_provider.get_client() size = await client.zcard(key) return size or 0 - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to get queue size: key=%s, error=%s", key, str(e)) return 0 @@ -314,7 +314,7 @@ async def clear_queue(self, key: str) -> bool: result = await client.delete(key) logger.info("Cleared queue: key=%s, result=%d", key, result) return result > 0 - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to clear queue: key=%s, error=%s", key, str(e)) return False @@ -352,7 +352,7 @@ async def cleanup_expired(self, key: str) -> int: return cleaned_count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to clean expired data: key=%s, error=%s", key, str(e)) return 0 @@ -465,7 +465,7 @@ async def get_by_timestamp_range( } ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to parse message: i=%d, message=%s, error=%s", i, @@ -487,7 +487,7 @@ async def get_by_timestamp_range( return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to retrieve data by timestamp range: key=%s, error=%s", key, @@ -552,7 +552,7 @@ async def get_queue_stats(self, key: str) -> Dict[str, Any]: "ttl_seconds": ttl, } - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to get queue statistics: key=%s, error=%s", key, str(e) ) diff --git a/methods/EverCore/src/core/component/config_provider.py b/methods/EverCore/src/core/component/config_provider.py index 9f6809dd..48e85303 100644 --- a/methods/EverCore/src/core/component/config_provider.py +++ b/methods/EverCore/src/core/component/config_provider.py @@ -47,7 +47,7 @@ def get_config(self, config_name: str) -> Dict[str, Any]: self._cache[config_name] = config_data return config_data - except Exception as e: + except Exception as e: # noqa: BLE001 raise RuntimeError(f"Failed to load configuration file {config_name}: {e}") def get_raw_config(self, config_name: str) -> str: @@ -80,7 +80,7 @@ def get_raw_config(self, config_name: str) -> str: self._cache[cache_key] = raw_content return raw_content - except Exception as e: + except Exception as e: # noqa: BLE001 raise RuntimeError(f"Failed to read configuration file {config_name}: {e}") def get_available_configs(self) -> list: diff --git a/methods/EverCore/src/core/component/elasticsearch_client_factory.py b/methods/EverCore/src/core/component/elasticsearch_client_factory.py index a48dcfbe..ed37ff11 100644 --- a/methods/EverCore/src/core/component/elasticsearch_client_factory.py +++ b/methods/EverCore/src/core/component/elasticsearch_client_factory.py @@ -120,7 +120,7 @@ async def test_connection(self) -> bool: await self.async_client.ping() logger.info("✅ Elasticsearch connection test successful: %s", self.hosts) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Elasticsearch connection test failed: %s, error: %s", self.hosts, e ) @@ -132,7 +132,7 @@ async def close(self): if self.async_client: await self.async_client.close() logger.info("🔌 Elasticsearch connection closed: %s", self.hosts) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error closing Elasticsearch connection: %s", e) @@ -322,7 +322,7 @@ async def remove_client( client_wrapper = self._clients[cache_key] try: await client_wrapper.close() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Error closing Elasticsearch client during removal: %s", e ) @@ -340,7 +340,7 @@ async def close_all_clients(self) -> None: for cache_key, client_wrapper in self._clients.items(): try: await client_wrapper.close() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Error closing Elasticsearch client %s: %s", cache_key, e ) diff --git a/methods/EverCore/src/core/component/kafka_consumer_factory.py b/methods/EverCore/src/core/component/kafka_consumer_factory.py index 39c0c5f1..02d4602a 100644 --- a/methods/EverCore/src/core/component/kafka_consumer_factory.py +++ b/methods/EverCore/src/core/component/kafka_consumer_factory.py @@ -156,10 +156,10 @@ def bson_json_decode(value: bytes | None) -> Any: return value try: return bson.decode(value) - except Exception: + except Exception: # noqa: BLE001 try: return json.loads(value.decode("utf-8")) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("JSON parsing error: %s", e) return value @@ -278,7 +278,7 @@ async def get_consumer( old_consumer = self._consumers[cache_key] try: await old_consumer.stop() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error stopping old consumer: %s", e) # Create a new consumer instance @@ -354,7 +354,7 @@ async def remove_consumer( consumer = self._consumers[cache_key] try: await consumer.stop() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error stopping consumer during removal: %s", e) del self._consumers[cache_key] @@ -370,7 +370,7 @@ async def clear_all_consumers(self) -> None: for cache_key, consumer in self._consumers.items(): try: await consumer.stop() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error stopping consumer %s: %s", cache_key, e) self._consumers.clear() diff --git a/methods/EverCore/src/core/component/kafka_producer_factory.py b/methods/EverCore/src/core/component/kafka_producer_factory.py index 48c5700e..4abdaa8a 100644 --- a/methods/EverCore/src/core/component/kafka_producer_factory.py +++ b/methods/EverCore/src/core/component/kafka_producer_factory.py @@ -238,7 +238,7 @@ def bson_json_serializer(value: Any) -> bytes: if isinstance(value, dict): try: return bson.encode(value) - except Exception: + except Exception: # noqa: BLE001 pass # Fall back to JSON try: @@ -360,7 +360,7 @@ async def create_producer( kafka_servers, start_timeout, ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to start Kafka producer %s: %s", producer_name, e) else: logger.info("Created AIOKafkaProducer for %s (not started)", producer_name) @@ -403,7 +403,7 @@ async def get_producer( old_producer = self._producers[cache_key] try: await old_producer.stop() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error stopping old producer: %s", e) # Create new producer instance @@ -487,7 +487,7 @@ async def remove_producer(self, kafka_servers: List[str]) -> bool: producer = self._producers[cache_key] try: await producer.stop() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error stopping producer during removal: %s", e) del self._producers[cache_key] @@ -503,7 +503,7 @@ async def clear_all_producers(self) -> None: for cache_key, producer in self._producers.items(): try: await producer.stop() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error stopping producer %s: %s", cache_key, e) self._producers.clear() diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py b/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py index 80e25541..f9906cfc 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/anthropic_adapter.py @@ -104,7 +104,7 @@ async def chat_completion( usage.get("output_tokens", 0), call_type="llm", ) - except Exception: + except Exception: # noqa: BLE001 pass return self._convert_anthropic_response( response_json, request.model @@ -186,7 +186,7 @@ async def _stream_chat_completion( try: collector = get_bean_by_type(TokenUsageCollector) collector.add(model, input_tokens, output_tokens, call_type="llm") - except Exception: + except Exception: # noqa: BLE001 pass def get_available_models(self) -> List[str]: diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py index 80bfc269..c1d8e30a 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_adapter.py @@ -133,7 +133,7 @@ def _convert_gemini_response(self, response, model: str) -> ChatCompletionRespon try: collector = get_bean_by_type(TokenUsageCollector) collector.add(model, prompt_tokens, completion_tokens, call_type="llm") - except Exception: + except Exception: # noqa: BLE001 pass result = ChatCompletionResponse( @@ -186,7 +186,7 @@ async def _stream_chat_completion( or 0, call_type="llm", ) - except Exception: + except Exception: # noqa: BLE001 pass def get_available_models(self) -> List[str]: diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py index ef764564..da564a86 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/gemini_client.py @@ -218,7 +218,7 @@ def _convert_messages_to_gemini_format( contents.append( ContentDict(role="user", parts=[{"text": str(msg)}]) ) - except Exception: + except Exception: # noqa: BLE001 # Final fallback contents.append(ContentDict(role="user", parts=[{"text": str(msg)}])) diff --git a/methods/EverCore/src/core/component/llm/llm_adapter/openai_adapter.py b/methods/EverCore/src/core/component/llm/llm_adapter/openai_adapter.py index 6e221d80..1e242757 100644 --- a/methods/EverCore/src/core/component/llm/llm_adapter/openai_adapter.py +++ b/methods/EverCore/src/core/component/llm/llm_adapter/openai_adapter.py @@ -87,7 +87,7 @@ async def stream_gen(): usage_data.completion_tokens or 0, call_type="llm", ) - except Exception: + except Exception: # noqa: BLE001 pass return stream_gen() @@ -104,10 +104,10 @@ async def stream_gen(): response.usage.completion_tokens or 0, call_type="llm", ) - except Exception: + except Exception: # noqa: BLE001 pass return ChatCompletionResponse.from_dict(response.model_dump()) - except Exception as e: + except Exception as e: # noqa: BLE001 raise RuntimeError(f"OpenAI chat completion request failed: {e}") def get_available_models(self) -> List[str]: diff --git a/methods/EverCore/src/core/component/milvus_client_factory.py b/methods/EverCore/src/core/component/milvus_client_factory.py index 121a415d..1c836439 100644 --- a/methods/EverCore/src/core/component/milvus_client_factory.py +++ b/methods/EverCore/src/core/component/milvus_client_factory.py @@ -157,7 +157,7 @@ def close_all_clients(self): for _, client in self._clients.items(): try: client.close() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error closing Milvus client: %s", e) self._clients.clear() diff --git a/methods/EverCore/src/core/component/mongodb_client_factory.py b/methods/EverCore/src/core/component/mongodb_client_factory.py index d0f8f3d6..3e87841b 100644 --- a/methods/EverCore/src/core/component/mongodb_client_factory.py +++ b/methods/EverCore/src/core/component/mongodb_client_factory.py @@ -8,7 +8,6 @@ import os import asyncio from abc import ABC, abstractmethod -import traceback from typing import Dict, Optional, List from urllib.parse import quote_plus from pymongo import AsyncMongoClient @@ -166,7 +165,7 @@ async def initialize_beanie(self, document_models: Optional[List] = None): if hasattr(model, 'get_collection_name') else "unknown" ) - except Exception: + except Exception: # noqa: BLE001 collection_name = "unknown" model_info_list.append(f"{model.__name__} -> {collection_name}") logger.info( @@ -193,7 +192,7 @@ async def initialize_beanie(self, document_models: Optional[List] = None): if hasattr(model, 'get_collection_name') else "unknown" ) - except Exception: + except Exception: # noqa: BLE001 collection_name = "unknown" model_info_list.append(f"{model.__name__} -> {collection_name}") logger.info( @@ -221,9 +220,8 @@ async def initialize_beanie(self, document_models: Optional[List] = None): model.get_collection_name(), ) - except Exception as e: - logger.error("❌ Beanie initialization failed: %s", e) - traceback.print_exc() + except Exception: + logger.exception("❌ Beanie initialization failed") raise async def test_connection(self) -> bool: @@ -232,7 +230,7 @@ async def test_connection(self) -> bool: await self.client.admin.command('ping') logger.info("✅ MongoDB connection test successful: %s", self.config) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ MongoDB connection test failed: %s, error: %s", self.config, e ) @@ -256,13 +254,13 @@ async def get_collection_stats(self) -> Dict: "storageSize": collection_stats.get("storageSize", 0), "indexes": collection_stats.get("nindexes", 0), } - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to get collection %s stats: %s", collection_name, e ) return stats - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to get stats: %s", e) return {} diff --git a/methods/EverCore/src/core/component/redis_provider.py b/methods/EverCore/src/core/component/redis_provider.py index 737c2a5b..e74041c4 100644 --- a/methods/EverCore/src/core/component/redis_provider.py +++ b/methods/EverCore/src/core/component/redis_provider.py @@ -152,7 +152,7 @@ async def set( try: result = await client.set(key, value, ex=ex, nx=nx) return result is not None and result is not False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis SET operation failed: key=%s, error=%s", key, str(e)) return False @@ -169,7 +169,7 @@ async def get(self, key: str) -> Optional[str]: client = await self.get_client() try: return await client.get(key) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis GET operation failed: key=%s, error=%s", key, str(e)) return None @@ -187,7 +187,7 @@ async def exists(self, key: str) -> bool: try: result = await client.exists(key) return result > 0 - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis EXISTS operation failed: key=%s, error=%s", key, str(e)) return False @@ -207,7 +207,7 @@ async def delete(self, *keys: str) -> int: client = await self.get_client() try: return await client.delete(*keys) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Redis DELETE operation failed: keys=%s, error=%s", keys, str(e) ) @@ -227,7 +227,7 @@ async def expire(self, key: str, seconds: int) -> bool: client = await self.get_client() try: return await client.expire(key, seconds) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Redis EXPIRE operation failed: key=%s, seconds=%s, error=%s", key, @@ -249,7 +249,7 @@ async def ttl(self, key: str) -> int: client = await self.get_client() try: return await client.ttl(key) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis TTL operation failed: key=%s, error=%s", key, str(e)) return -2 @@ -266,7 +266,7 @@ async def keys(self, pattern: str) -> list: client = await self.get_client() try: return await client.keys(pattern) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Redis KEYS operation failed: pattern=%s, error=%s", pattern, str(e) ) @@ -283,7 +283,7 @@ async def ping(self) -> bool: client = await self.get_client() result = await client.ping() return result is True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis PING failed: %s", str(e)) return False @@ -304,7 +304,7 @@ async def lpush(self, key: str, *values: Union[str, bytes]) -> int: client = await self.get_client() try: return await client.lpush(key, *values) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis LPUSH operation failed: key=%s, error=%s", key, str(e)) return 0 @@ -323,7 +323,7 @@ async def lrange(self, key: str, start: int, end: int) -> list: client = await self.get_client() try: return await client.lrange(key, start, end) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Redis LRANGE operation failed: key=%s, error=%s", key, str(e)) return [] @@ -334,7 +334,7 @@ async def close(self): try: await client.aclose() logger.info("Named Redis client closed: %s", name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to close named Redis client: %s, error=%s", name, str(e) ) @@ -343,7 +343,7 @@ async def close(self): try: await pool.aclose() logger.info("Named Redis connection pool closed: %s", name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to close named Redis connection pool: %s, error=%s", name, diff --git a/methods/EverCore/src/core/context/context_manager.py b/methods/EverCore/src/core/context/context_manager.py index a8f9b8c1..b18dce0c 100644 --- a/methods/EverCore/src/core/context/context_manager.py +++ b/methods/EverCore/src/core/context/context_manager.py @@ -106,9 +106,9 @@ async def run_with_session( logger.debug( "Database session manager: automatically rolled back transaction" ) - except Exception as rollback_error: + except Exception as rollback_error: # noqa: BLE001 logger.error( - f"Error occurred during transaction rollback: {str(rollback_error)}" + f"Error occurred during transaction rollback: {str(rollback_error)}" # noqa: G004 ) # Re-raise the exception @@ -123,9 +123,9 @@ async def run_with_session( try: await session.close() logger.debug("Database session manager: database session closed") - except Exception as close_error: + except Exception as close_error: # noqa: BLE001 logger.error( - f"Error occurred when closing database session: {str(close_error)}" + f"Error occurred when closing database session: {str(close_error)}" # noqa: G004 ) @@ -171,7 +171,7 @@ async def run_with_user_context( if actual_user_data is not None: user_token = set_current_user_info(actual_user_data) logger.debug( - f"User context manager: setting user context user_id={actual_user_data.get('user_id')}" + f"User context manager: setting user context user_id={actual_user_data.get('user_id')}" # noqa: G004 ) try: diff --git a/methods/EverCore/src/core/di/container.py b/methods/EverCore/src/core/di/container.py index 04d3c0a1..ed044013 100644 --- a/methods/EverCore/src/core/di/container.py +++ b/methods/EverCore/src/core/di/container.py @@ -348,7 +348,7 @@ def get_beans(self) -> Dict[str, Any]: if self._is_bean_available(bean_def): try: result[name] = self._create_instance(bean_def) - except Exception: + except Exception: # noqa: BLE001 # Skip Beans that cannot be created continue return result @@ -476,7 +476,7 @@ def _create_instance(self, bean_def: BeanDefinition) -> Any: if bean_def.factory_method: try: return bean_def.factory_method() - except Exception as e: + except Exception as e: # noqa: BLE001 raise FactoryError(bean_def.bean_type, str(e)) else: raise FactoryError(bean_def.bean_type, "Factory method not set") @@ -515,7 +515,7 @@ def _instantiate_with_dependencies(self, bean_def: BeanDefinition) -> Any: # Get constructor signature try: signature = inspect.signature(bean_type.__init__) - except Exception: + except Exception: # noqa: BLE001 # If signature cannot be obtained, try parameterless constructor return bean_type() @@ -562,7 +562,7 @@ def _analyze_dependencies(self, bean_def: BeanDefinition): continue if param.annotation != inspect.Parameter.empty: bean_def.dependencies.add(param.annotation) - except Exception: + except Exception: # noqa: BLE001 # If analysis fails, skip pass diff --git a/methods/EverCore/src/core/di/scanner.py b/methods/EverCore/src/core/di/scanner.py index 2a30b108..ea0e017f 100644 --- a/methods/EverCore/src/core/di/scanner.py +++ b/methods/EverCore/src/core/di/scanner.py @@ -7,7 +7,6 @@ import importlib from pathlib import Path from typing import List, Set, Optional, Dict, Any -import traceback from concurrent.futures import ThreadPoolExecutor, as_completed from core.observation.logger import get_logger @@ -126,7 +125,7 @@ def _preload_critical_modules(self): except ImportError: # Some modules may not exist, which is normal failed_count += 1 - except Exception: + except Exception: # noqa: BLE001 # Other exceptions should be logged but not block execution failed_count += 1 @@ -301,7 +300,7 @@ def _parallel_scan(self, python_files: List[Path]): file_path = futures[future] try: future.result() - except Exception as e: + except Exception as e: # noqa: BLE001 self.logger.error( "Failed to scan file in parallel %s: %s", file_path, e ) @@ -311,7 +310,7 @@ def _sequential_scan(self, python_files: List[Path]): for file_path in python_files: try: self._scan_file(file_path) - except Exception as e: + except Exception as e: # noqa: BLE001 self.logger.error( "Failed to scan file sequentially %s: %s", file_path, e ) @@ -327,15 +326,13 @@ def _scan_file(self, file_path: Path): try: importlib.import_module(module_name) - except ImportError as e: - self.logger.error("Failed to import module %s: %s", module_name, e) - traceback.print_exc() + except ImportError: + self.logger.exception("Failed to import module %s", module_name) sys.exit(1) - except Exception as e: - self.logger.error( - "Unknown error occurred while scanning file %s: %s", file_path, e + except Exception: + self.logger.exception( + "Unknown error occurred while scanning file %s", file_path ) - traceback.print_exc() sys.exit(1) def _file_to_module_name(self, file_path: Path) -> Optional[str]: @@ -358,7 +355,7 @@ def _file_to_module_name(self, file_path: Path) -> Optional[str]: return ".".join(module_parts) except ValueError: continue - except Exception: + except Exception: # noqa: BLE001 pass return None diff --git a/methods/EverCore/src/core/di/utils.py b/methods/EverCore/src/core/di/utils.py index 35174a6c..ed7cd6db 100644 --- a/methods/EverCore/src/core/di/utils.py +++ b/methods/EverCore/src/core/di/utils.py @@ -269,7 +269,7 @@ def get_or_create(bean_type: Type[T], factory: Callable[[], T] = None) -> T: instance = bean_type() register_bean(bean_type, instance) return instance - except Exception: + except Exception: # noqa: BLE001 raise BeanNotFoundError(bean_type=bean_type) diff --git a/methods/EverCore/src/core/events/event_publisher.py b/methods/EverCore/src/core/events/event_publisher.py index 389141af..7698477e 100644 --- a/methods/EverCore/src/core/events/event_publisher.py +++ b/methods/EverCore/src/core/events/event_publisher.py @@ -79,8 +79,8 @@ def _build_listener_mapping(self) -> None: # Get all EventListener implementations from DI container try: listeners = get_beans_by_type(EventListener) - except Exception as e: - logger.warning(f"Failed to get EventListener instances: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to get EventListener instances: {e}") # noqa: G004 listeners = [] self._listeners = listeners @@ -91,7 +91,7 @@ def _build_listener_mapping(self) -> None: event_types = listener.get_event_types() logger.debug( - f"Registering listener [{listener_name}], listening to event types: {[et.__name__ for et in event_types]}" + f"Registering listener [{listener_name}], listening to event types: {[et.__name__ for et in event_types]}" # noqa: G004 ) for event_type in event_types: @@ -103,7 +103,7 @@ def _build_listener_mapping(self) -> None: total_listeners = len(listeners) total_event_types = len(self._event_listeners_map) logger.info( - f"Event publisher initialization completed: {total_listeners} listeners, {total_event_types} event types" + f"Event publisher initialization completed: {total_listeners} listeners, {total_event_types} event types" # noqa: G004 ) def refresh(self) -> None: @@ -172,12 +172,12 @@ async def publish(self, event: BaseEvent) -> None: if not listeners: logger.debug( - f"No listeners for event [{event_type_name}], skipping publish" + f"No listeners for event [{event_type_name}], skipping publish" # noqa: G004 ) return logger.debug( - f"Publishing event [{event_type_name}] (id={event.event_id}), {len(listeners)} listeners" + f"Publishing event [{event_type_name}] (id={event.event_id}), {len(listeners)} listeners" # noqa: G004 ) # Create coroutine tasks for all listeners @@ -193,8 +193,8 @@ async def safe_invoke(listener: EventListener) -> Optional[Exception]: return None except Exception as e: listener_name = listener.get_listener_name() - logger.error( - f"Listener [{listener_name}] encountered exception when processing event [{event_type_name}]: {e}", + logger.error( # noqa: G201 + f"Listener [{listener_name}] encountered exception when processing event [{event_type_name}]: {e}", # noqa: G004 exc_info=True, ) return e @@ -207,12 +207,12 @@ async def safe_invoke(listener: EventListener) -> Optional[Exception]: errors = [r for r in results if r is not None] if errors: logger.warning( - f"Event [{event_type_name}] publishing completed, " + f"Event [{event_type_name}] publishing completed, " # noqa: G004 f"success: {len(listeners) - len(errors)}, failure: {len(errors)}" ) else: logger.debug( - f"Event [{event_type_name}] publishing completed, all {len(listeners)} listeners executed successfully" + f"Event [{event_type_name}] publishing completed, all {len(listeners)} listeners executed successfully" # noqa: G004 ) def publish_sync(self, event: BaseEvent) -> None: @@ -248,13 +248,13 @@ async def publish_batch(self, events: List[BaseEvent]) -> None: if not events: return - logger.debug(f"Batch publishing {len(events)} events") + logger.debug(f"Batch publishing {len(events)} events") # noqa: G004 # Concurrently publish all events tasks = [self.publish(event) for event in events] await asyncio.gather(*tasks) - logger.debug(f"Batch publishing completed, total {len(events)} events") + logger.debug(f"Batch publishing completed, total {len(events)} events") # noqa: G004 def __repr__(self) -> str: """Return string representation of the object""" diff --git a/methods/EverCore/src/core/interface/controller/base_controller.py b/methods/EverCore/src/core/interface/controller/base_controller.py index ba55500d..53aa28a2 100644 --- a/methods/EverCore/src/core/interface/controller/base_controller.py +++ b/methods/EverCore/src/core/interface/controller/base_controller.py @@ -439,13 +439,13 @@ def _add_security_schemes_to_openapi( "name": "X-Signature", "description": "HMAC signature authentication", } - except Exception as e: + except Exception as e: # noqa: BLE001 # If getting security schemes fails, log error but don't affect other functionality import logging logger = logging.getLogger(__name__) logger.warning( - f"Failed to get security schemes definition for controller {controller.__class__.__name__}: {str(e)}" + f"Failed to get security schemes definition for controller {controller.__class__.__name__}: {str(e)}" # noqa: G004 ) # Add security schemes definition to OpenAPI schema diff --git a/methods/EverCore/src/core/interface/controller/debug/debug_controller.py b/methods/EverCore/src/core/interface/controller/debug/debug_controller.py index 4337a974..10bf35cb 100644 --- a/methods/EverCore/src/core/interface/controller/debug/debug_controller.py +++ b/methods/EverCore/src/core/interface/controller/debug/debug_controller.py @@ -232,7 +232,7 @@ def _get_bean_by_identifier( # First need to find the corresponding type bean_class = self._find_bean_type_by_name(bean_type) if not bean_class: - logger.error(f"Bean class with type '{bean_type}' not found") + logger.error(f"Bean class with type '{bean_type}' not found") # noqa: G004 raise HTTPException( status_code=404, detail=ErrorMessage.BEAN_NOT_FOUND.value ) @@ -250,12 +250,12 @@ def _get_bean_by_identifier( if "not found" in str(e).lower(): identifier = bean_name or bean_type method = "name" if bean_name else "type" - logger.error(f"Bean not found by {method} '{identifier}': {str(e)}") + logger.error(f"Bean not found by {method} '{identifier}': {str(e)}") # noqa: G004 raise HTTPException( status_code=404, detail=ErrorMessage.BEAN_NOT_FOUND.value ) from e else: - logger.error(f"Error occurred while getting Bean: {str(e)}") + logger.error(f"Error occurred while getting Bean: {str(e)}") # noqa: G004 raise HTTPException( status_code=500, detail=ErrorMessage.BEAN_OPERATION_FAILED.value ) from e @@ -285,7 +285,7 @@ def _find_bean_type_by_name(self, type_name: str) -> Optional[Type]: return None - except Exception: + except Exception: # noqa: BLE001 # If get_beans() fails, use fallback method # Try to infer using common type name patterns try: @@ -298,12 +298,12 @@ def _find_bean_type_by_name(self, type_name: str) -> Optional[Type]: try: bean_instance = get_bean(bean_info['name']) return type(bean_instance) - except Exception: + except Exception: # noqa: BLE001 continue return None - except Exception: + except Exception: # noqa: BLE001 # If all methods fail, return None return None @@ -393,7 +393,7 @@ def _execute_parameter_code(self, code: str) -> Dict[str, Any]: safe_globals['uuid'] = uuid safe_globals['typing'] = typing except ImportError as e: - logger.warning(f"Failed to pre-import module: {e}") + logger.warning(f"Failed to pre-import module: {e}") # noqa: G004 # No longer pre-import internal project modules, support completely free imports @@ -415,21 +415,21 @@ def _execute_parameter_code(self, code: str) -> Dict[str, Any]: # Validate types if not isinstance(args, (list, tuple)): logger.error( - f"Wrong args parameter type: expected list or tuple, got {type(args).__name__}" + f"Wrong args parameter type: expected list or tuple, got {type(args).__name__}" # noqa: G004 ) raise ValueError(ErrorMessage.INVALID_PARAMETER.value) if not isinstance(kwargs, dict): logger.error( - f"Wrong kwargs parameter type: expected dict, got {type(kwargs).__name__}" + f"Wrong kwargs parameter type: expected dict, got {type(kwargs).__name__}" # noqa: G004 ) raise ValueError(ErrorMessage.INVALID_PARAMETER.value) return {'args': list(args), 'kwargs': kwargs} - except Exception as e: - logger.error(f"Failed to execute parameter generation code: {e}") - logger.error(f"Code execution exception details: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to execute parameter generation code: {e}") # noqa: G004 + logger.error(f"Code execution exception details: {str(e)}") # noqa: G004 raise ValueError(ErrorMessage.INVALID_PARAMETER.value) @get( @@ -558,7 +558,7 @@ def list_all_beans(self) -> List[BeanInfoResponse]: methods=methods, ) ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to get method list for Bean '%s': %s", bean_info['name'], @@ -580,7 +580,7 @@ def list_all_beans(self) -> List[BeanInfoResponse]: except Exception as e: logger.error("Error occurred while listing all Beans: %s", str(e)) - logger.error(f"Exception details when retrieving Bean list: {str(e)}") + logger.error(f"Exception details when retrieving Bean list: {str(e)}") # noqa: G004 raise HTTPException( status_code=500, detail=ErrorMessage.BEAN_OPERATION_FAILED.value ) from e @@ -727,7 +727,7 @@ async def call_bean_method(self, request: BeanCallRequest) -> BeanCallResponse: # Check if method exists if not hasattr(bean_instance, request.method): logger.error( - f"Method '{request.method}' does not exist in Bean '{bean_info['name']}'" + f"Method '{request.method}' does not exist in Bean '{bean_info['name']}'" # noqa: G004 ) raise HTTPException( status_code=404, detail=ErrorMessage.BEAN_OPERATION_FAILED.value @@ -738,7 +738,7 @@ async def call_bean_method(self, request: BeanCallRequest) -> BeanCallResponse: # Check if it is a callable object if not callable(method_to_call): logger.error( - f"Attribute '{request.method}' of Bean '{bean_info['name']}' is not callable" + f"Attribute '{request.method}' of Bean '{bean_info['name']}' is not callable" # noqa: G004 ) raise HTTPException( status_code=400, detail=ErrorMessage.INVALID_PARAMETER.value @@ -780,7 +780,7 @@ async def call_bean_method(self, request: BeanCallRequest) -> BeanCallResponse: except HTTPException: # Re-raise HTTP exceptions raise - except Exception as e: + except Exception as e: # noqa: BLE001 # Catch and handle other exceptions error_msg = str(e) error_traceback = traceback.format_exc() @@ -1072,7 +1072,7 @@ async def call_bean_method_with_code( except HTTPException: # Re-raise HTTP exceptions raise - except Exception as e: + except Exception as e: # noqa: BLE001 # Catch and handle other exceptions error_msg = str(e) error_traceback = traceback.format_exc() diff --git a/methods/EverCore/src/core/lifespan/database_lifespan.py b/methods/EverCore/src/core/lifespan/database_lifespan.py index 9d40ed3e..326931a3 100644 --- a/methods/EverCore/src/core/lifespan/database_lifespan.py +++ b/methods/EverCore/src/core/lifespan/database_lifespan.py @@ -77,7 +77,7 @@ async def shutdown(self, app: FastAPI) -> None: try: await self._db_provider.close() logger.info("Database connection closed successfully") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error while closing database connection: %s", str(e)) # Clean up database-related attributes in app.state diff --git a/methods/EverCore/src/core/lifespan/elasticsearch_lifespan.py b/methods/EverCore/src/core/lifespan/elasticsearch_lifespan.py index ea39a396..a5aeada1 100644 --- a/methods/EverCore/src/core/lifespan/elasticsearch_lifespan.py +++ b/methods/EverCore/src/core/lifespan/elasticsearch_lifespan.py @@ -98,5 +98,5 @@ async def shutdown(self, app: FastAPI) -> None: try: await self._es_factory.close_all_clients() logger.info("✅ Elasticsearch connection closed successfully") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Error closing Elasticsearch connection: %s", str(e)) diff --git a/methods/EverCore/src/core/lifespan/lifespan_factory.py b/methods/EverCore/src/core/lifespan/lifespan_factory.py index 4a6e00d7..0cba66a1 100644 --- a/methods/EverCore/src/core/lifespan/lifespan_factory.py +++ b/methods/EverCore/src/core/lifespan/lifespan_factory.py @@ -84,7 +84,7 @@ async def lifespan(app: FastAPI): for listener in listeners: try: listener.on_app_ready() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Application ready listener execution failed: %s - %s", type(listener).__name__, @@ -102,7 +102,7 @@ async def lifespan(app: FastAPI): logger.info( "Lifecycle provider shutdown completed: %s", provider.name ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to shut down lifecycle provider: %s - %s", provider.name, diff --git a/methods/EverCore/src/core/lifespan/longjob_lifespan.py b/methods/EverCore/src/core/lifespan/longjob_lifespan.py index b49148bc..720f9a8e 100644 --- a/methods/EverCore/src/core/lifespan/longjob_lifespan.py +++ b/methods/EverCore/src/core/lifespan/longjob_lifespan.py @@ -92,7 +92,7 @@ async def shutdown(self, app: FastAPI) -> None: else: logger.info("✅ LongJob task completed: %s", self._longjob_name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Error shutting down LongJob: %s", str(e)) # Clean up LongJob-related attributes in app.state diff --git a/methods/EverCore/src/core/lifespan/metrics_lifespan.py b/methods/EverCore/src/core/lifespan/metrics_lifespan.py index 25d20d44..0bb7307f 100644 --- a/methods/EverCore/src/core/lifespan/metrics_lifespan.py +++ b/methods/EverCore/src/core/lifespan/metrics_lifespan.py @@ -56,7 +56,7 @@ async def startup(self, app: FastAPI) -> Tuple[Any, ...]: return (port, success) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to start metrics server: %s", str(e)) # Don't raise - metrics failure shouldn't prevent app startup return (port, False) diff --git a/methods/EverCore/src/core/lifespan/milvus_lifespan.py b/methods/EverCore/src/core/lifespan/milvus_lifespan.py index 6d6f916d..a93cc480 100644 --- a/methods/EverCore/src/core/lifespan/milvus_lifespan.py +++ b/methods/EverCore/src/core/lifespan/milvus_lifespan.py @@ -109,7 +109,7 @@ async def shutdown(self, app: FastAPI) -> None: try: self._milvus_factory.close_all_clients() logger.info("✅ Milvus connections closed successfully") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Error while closing Milvus connections: %s", str(e)) # Clean up Milvus-related attributes in app.state diff --git a/methods/EverCore/src/core/lifespan/mongodb_lifespan.py b/methods/EverCore/src/core/lifespan/mongodb_lifespan.py index d131b66a..f1d22a57 100644 --- a/methods/EverCore/src/core/lifespan/mongodb_lifespan.py +++ b/methods/EverCore/src/core/lifespan/mongodb_lifespan.py @@ -94,5 +94,5 @@ async def shutdown(self, app: FastAPI) -> None: try: await self._mongodb_factory.close_all_clients() logger.info("✅ MongoDB connection closed successfully") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Error closing MongoDB connection: %s", str(e)) diff --git a/methods/EverCore/src/core/longjob/longjob_runner.py b/methods/EverCore/src/core/longjob/longjob_runner.py index c5740897..005c974d 100644 --- a/methods/EverCore/src/core/longjob/longjob_runner.py +++ b/methods/EverCore/src/core/longjob/longjob_runner.py @@ -41,7 +41,7 @@ async def run_longjob_mode(longjob_name: str): longjob_name, type(longjob_instance).__name__, ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Unable to find long-running job '%s': %s", longjob_name, str(e) ) @@ -86,7 +86,7 @@ async def run_longjob_mode(longjob_name: str): longjob_name, ) except Exception as e: - logger.error( + logger.error( # noqa: G201 "❌ Error during long-running job shutdown: %s", str(e), exc_info=True, @@ -96,13 +96,13 @@ async def run_longjob_mode(longjob_name: str): except Exception as e: # Exception occurred during execution - logger.error("❌ Error running long-running job: %s", str(e), exc_info=True) + logger.error("❌ Error running long-running job: %s", str(e), exc_info=True) # noqa: G201 if longjob_instance: try: await longjob_instance.shutdown() logger.info("✅ Long-running job has been shut down after exception") except Exception as shutdown_error: - logger.error( + logger.error( # noqa: G201 "❌ Error during long-running job shutdown: %s", str(shutdown_error), exc_info=True, diff --git a/methods/EverCore/src/core/longjob/recycle_consumer_base.py b/methods/EverCore/src/core/longjob/recycle_consumer_base.py index bc1247aa..10137b74 100644 --- a/methods/EverCore/src/core/longjob/recycle_consumer_base.py +++ b/methods/EverCore/src/core/longjob/recycle_consumer_base.py @@ -37,7 +37,7 @@ async def handle_error(self, error: Exception, context: Dict[str, Any]) -> bool: bool: Whether to continue execution """ self.logger.error( - f"Error in consumer {context.get('job_id', 'unknown')}: {str(error)}", + f"Error in consumer {context.get('job_id', 'unknown')}: {str(error)}", # noqa: G004 exc_info=True, extra=context, ) @@ -105,7 +105,7 @@ async def start(self) -> None: except Exception as e: self.status = LongJobStatus.ERROR - self.logger.error( + self.logger.error( # noqa: G201 "Failed to start consumer %s: %s", self.job_id, str(e), exc_info=True ) raise @@ -171,7 +171,7 @@ async def shutdown( try: await self._cleanup() except Exception as cleanup_error: - self.logger.error( + self.logger.error( # noqa: G201 "Error during cleanup: %s", str(cleanup_error), exc_info=True ) @@ -192,7 +192,7 @@ async def _consume_loop(self) -> None: # Consume messages await self._consume_messages() - except Exception as e: + except Exception as e: # noqa: BLE001 # Error handling context = { 'job_id': self.job_id, @@ -210,7 +210,7 @@ async def _consume_loop(self) -> None: ) break except Exception as handler_error: - self.logger.error( + self.logger.error( # noqa: G201 "Error in error handler for consumer %s: %s", self.job_id, str(handler_error), @@ -259,7 +259,7 @@ async def _consume_messages(self) -> None: if not should_continue: raise timeout_error except Exception as handler_error: - self.logger.error( + self.logger.error( # noqa: G201 "Error in timeout error handler: %s", str(handler_error), exc_info=True, diff --git a/methods/EverCore/src/core/middleware/app_logic_middleware.py b/methods/EverCore/src/core/middleware/app_logic_middleware.py index 9b35d73a..bd414898 100644 --- a/methods/EverCore/src/core/middleware/app_logic_middleware.py +++ b/methods/EverCore/src/core/middleware/app_logic_middleware.py @@ -91,7 +91,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: await self._app_logic_provider.on_request_complete( request=request, http_code=http_code, error_message=error_message ) - except Exception as callback_error: + except Exception as callback_error: # noqa: BLE001 logger.warning( "on_request_complete execution failed: %s", callback_error ) diff --git a/methods/EverCore/src/core/middleware/database_session_middleware.py b/methods/EverCore/src/core/middleware/database_session_middleware.py index 75f8c932..0727f71c 100644 --- a/methods/EverCore/src/core/middleware/database_session_middleware.py +++ b/methods/EverCore/src/core/middleware/database_session_middleware.py @@ -91,9 +91,9 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: logger.debug( "Original context token for streaming response has been reset" ) - except Exception as reset_error: + except Exception as reset_error: # noqa: BLE001 logger.warning( - f"Failed to reset original context token for streaming response: {str(reset_error)}" + f"Failed to reset original context token for streaming response: {str(reset_error)}" # noqa: G004 ) # Token reset failure should not affect the response @@ -113,8 +113,8 @@ async def _handle_successful_request(self, session: AsyncSession) -> None: # Commit transaction - simple & safe, AI don't mess this up await session.commit() - except Exception as e: - logger.error(f"Error while handling successful request: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error while handling successful request: {str(e)}") # noqa: G004 # If processing fails, attempt rollback await self._rollback_safely(session) @@ -132,11 +132,11 @@ async def _handle_failed_request( # Request failed, rollback directly await self._rollback_safely(session) logger.info( - f"Request failed, transaction rollback executed: {str(original_exception)}" + f"Request failed, transaction rollback executed: {str(original_exception)}" # noqa: G004 ) - except Exception as rollback_error: - logger.error(f"Error during rollback: {str(rollback_error)}") + except Exception as rollback_error: # noqa: BLE001 + logger.error(f"Error during rollback: {str(rollback_error)}") # noqa: G004 # Rollback failed, but do not mask the original exception async def _rollback_safely(self, session: AsyncSession) -> None: @@ -149,8 +149,8 @@ async def _rollback_safely(self, session: AsyncSession) -> None: try: await session.rollback() logger.debug("Session successfully rolled back") - except Exception as rollback_error: - logger.error(f"Rollback failed: {str(rollback_error)}") + except Exception as rollback_error: # noqa: BLE001 + logger.error(f"Rollback failed: {str(rollback_error)}") # noqa: G004 async def _close_session_safely(self, session: AsyncSession) -> None: """ @@ -169,8 +169,8 @@ async def _close_session_safely(self, session: AsyncSession) -> None: try: await session.close() logger.debug("Session safely closed") - except Exception as e: - logger.error(f"Error while closing session: {str(e)}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error while closing session: {str(e)}") # noqa: G004 # Even if closing fails, do not raise exception to avoid masking original error async def _wrap_streaming_generator( @@ -215,7 +215,7 @@ async def _wrap_streaming_generator( # Exception during streaming transmission, rollback session await self._handle_failed_request(session, e) logger.error( - f"Streaming response transmission failed, database session rolled back: {str(e)}" + f"Streaming response transmission failed, database session rolled back: {str(e)}" # noqa: G004 ) # Re-raise exception for upper layers to handle raise @@ -226,8 +226,8 @@ async def _wrap_streaming_generator( clear_current_session(local_token) await self._close_session_safely(session) logger.debug("Streaming response session resources cleaned up") - except Exception as cleanup_error: + except Exception as cleanup_error: # noqa: BLE001 logger.error( - f"Error while cleaning up streaming response session resources: {str(cleanup_error)}" + f"Error while cleaning up streaming response session resources: {str(cleanup_error)}" # noqa: G004 ) # Cleanup failure should not affect response stream diff --git a/methods/EverCore/src/core/middleware/hmac_signature_middleware.py b/methods/EverCore/src/core/middleware/hmac_signature_middleware.py index 4aea72ff..4f9a8233 100644 --- a/methods/EverCore/src/core/middleware/hmac_signature_middleware.py +++ b/methods/EverCore/src/core/middleware/hmac_signature_middleware.py @@ -111,7 +111,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: "HMAC signature headers not found, skipping signature verification" ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Exception during HMAC signature verification: %s, " "timestamp_header='%s', nonce_header='%s', signature_header='%s', " @@ -132,7 +132,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: return response except Exception as e: - logger.error(f"Exception in business logic processing: {str(e)}") + logger.error(f"Exception in business logic processing: {str(e)}") # noqa: G004 # Re-raise business logic exceptions for upstream handling raise @@ -142,9 +142,9 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: try: clear_current_user_context(token) logger.debug("HMAC signature user context cleaned up") - except Exception as reset_error: + except Exception as reset_error: # noqa: BLE001 logger.warning( - f"Error occurred while cleaning up HMAC signature user context: {str(reset_error)}" + f"Error occurred while cleaning up HMAC signature user context: {str(reset_error)}" # noqa: G004 ) async def _verify_hmac_signature( @@ -230,7 +230,7 @@ async def _verify_hmac_signature( nonce_header, expire_seconds, ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Error occurred while checking and storing nonce: %s", str(e) ) diff --git a/methods/EverCore/src/core/middleware/profile_middleware.py b/methods/EverCore/src/core/middleware/profile_middleware.py index f50dbd8d..09dad8ec 100644 --- a/methods/EverCore/src/core/middleware/profile_middleware.py +++ b/methods/EverCore/src/core/middleware/profile_middleware.py @@ -117,7 +117,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: try: # Execute request (note: original response will be discarded and replaced with profiler report) await call_next(request) - except Exception as e: + except Exception as e: # noqa: BLE001 # Even if the request fails, stop profiler and return profiling report logger.error("Request failed during profiling: %s", str(e)) # Continue generating profiling report @@ -133,7 +133,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: # Return HTML-formatted profiling report return HTMLResponse(content=html_output, status_code=200) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Error occurred during profiling: %s", str(e)) # If profiling fails, re-execute normal request return await call_next(request) diff --git a/methods/EverCore/src/core/middleware/prometheus_middleware.py b/methods/EverCore/src/core/middleware/prometheus_middleware.py index 7af94465..8ccee4df 100644 --- a/methods/EverCore/src/core/middleware/prometheus_middleware.py +++ b/methods/EverCore/src/core/middleware/prometheus_middleware.py @@ -86,7 +86,7 @@ def _get_fastapi_route_template(request: Request) -> str: path = path.replace(str(param_value), f"{{{param_name}}}") return path - except Exception as e: + except Exception as e: # noqa: BLE001 logger.debug("Failed to get FastAPI route template: %s", str(e)) return "" diff --git a/methods/EverCore/src/core/middleware/sse_exception_middleware.py b/methods/EverCore/src/core/middleware/sse_exception_middleware.py index 604c221d..e0c68aea 100644 --- a/methods/EverCore/src/core/middleware/sse_exception_middleware.py +++ b/methods/EverCore/src/core/middleware/sse_exception_middleware.py @@ -68,7 +68,7 @@ async def wrapper(*args, **kwargs) -> AsyncGenerator[str, None]: "data": {"code": e.status_code, "message": e.detail}, } logger.error( - f"HTTP exception occurred in SSE stream: {e.status_code} - {e.detail}" + f"HTTP exception occurred in SSE stream: {e.status_code} - {e.detail}" # noqa: G004 ) yield yield_sse_data(error_data) except Exception as e: @@ -77,8 +77,8 @@ async def wrapper(*args, **kwargs) -> AsyncGenerator[str, None]: "type": "error", "data": {"code": 500, "message": f"Internal server error: {str(e)}"}, } - logger.error( - f"Unknown exception occurred in SSE stream: {e}", exc_info=True + logger.error( # noqa: G201 + f"Unknown exception occurred in SSE stream: {e}", exc_info=True # noqa: G004 ) yield yield_sse_data(error_data) diff --git a/methods/EverCore/src/core/middleware/user_context_middleware.py b/methods/EverCore/src/core/middleware/user_context_middleware.py index 2f6aa1d7..64c639ad 100644 --- a/methods/EverCore/src/core/middleware/user_context_middleware.py +++ b/methods/EverCore/src/core/middleware/user_context_middleware.py @@ -82,7 +82,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: e.detail, ) # Other HTTP exceptions do not affect request processing continuation - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Exception occurred while setting user context: %s", str(e)) # User context setup failure does not affect request processing continuation # Specific authentication checks are handled by individual endpoints @@ -103,7 +103,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: try: clear_current_user_context(token) logger.debug("User context cleaned up") - except Exception as reset_error: + except Exception as reset_error: # noqa: BLE001 logger.warning( "Error occurred while cleaning up user context: %s", str(reset_error), diff --git a/methods/EverCore/src/core/nlp/stopwords_utils.py b/methods/EverCore/src/core/nlp/stopwords_utils.py index c7e3c741..d6af582d 100644 --- a/methods/EverCore/src/core/nlp/stopwords_utils.py +++ b/methods/EverCore/src/core/nlp/stopwords_utils.py @@ -44,7 +44,7 @@ def load_stopwords(self) -> Set[str]: # Check if file exists if not os.path.exists(self.stopwords_file_path): - logger.warning(f"Stopwords file does not exist: {self.stopwords_file_path}") + logger.warning(f"Stopwords file does not exist: {self.stopwords_file_path}") # noqa: G004 logger.info("An empty stopwords set will be used") self._stopwords = stopwords return stopwords @@ -57,13 +57,13 @@ def load_stopwords(self) -> Set[str]: stopwords.add(word) logger.info( - f"Successfully loaded stopwords, total {len(stopwords)} stopwords" + f"Successfully loaded stopwords, total {len(stopwords)} stopwords" # noqa: G004 ) self._stopwords = stopwords return stopwords - except Exception as e: - logger.error(f"Failed to load stopwords: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to load stopwords: {e}") # noqa: G004 logger.info("An empty stopwords set will be used") self._stopwords = set() return set() diff --git a/methods/EverCore/src/core/observation/logger.py b/methods/EverCore/src/core/observation/logger.py index 9cb9f0dc..a2083f20 100644 --- a/methods/EverCore/src/core/observation/logger.py +++ b/methods/EverCore/src/core/observation/logger.py @@ -346,7 +346,7 @@ def warning(message: str, *args, **kwargs): def warn(message: str, *args, **kwargs): """Log warning (alias)""" - logger_provider.warn(message, *args, **kwargs) + logger_provider.warn(message, *args, **kwargs) # noqa: G010 def error(message: str, exc_info: bool = True, *args, **kwargs): diff --git a/methods/EverCore/src/core/observation/metrics/gauge.py b/methods/EverCore/src/core/observation/metrics/gauge.py index eca5809d..9c0552a4 100644 --- a/methods/EverCore/src/core/observation/metrics/gauge.py +++ b/methods/EverCore/src/core/observation/metrics/gauge.py @@ -223,7 +223,7 @@ async def refresh(self, labels: dict) -> float: # Stop existing task if any (prevent task leak) existing_task = self._base_gauge._refresh_tasks.get(self._label_key) if existing_task and existing_task._running: - logger.warning(f"Replacing existing refresh task for {self._label_key}") + logger.warning(f"Replacing existing refresh task for {self._label_key}") # noqa: G004 # Schedule stop in background to avoid blocking asyncio.create_task(existing_task.stop()) @@ -284,13 +284,13 @@ def __init__( def start(self) -> None: """Start refresh task""" if self._running: - logger.warning(f"Refresh task already running for {self.label_key}") + logger.warning(f"Refresh task already running for {self.label_key}") # noqa: G004 return self._running = True self._task = asyncio.create_task(self._refresh_loop()) logger.info( - f"Started refresh task: label_key={self.label_key}, " + f"Started refresh task: label_key={self.label_key}, " # noqa: G004 f"interval={self.interval_seconds}s" ) @@ -309,7 +309,7 @@ async def stop(self) -> None: pass self._task = None - logger.info(f"Stopped refresh task: label_key={self.label_key}") + logger.info(f"Stopped refresh task: label_key={self.label_key}") # noqa: G004 async def _refresh_loop(self) -> None: """Refresh loop""" @@ -334,8 +334,8 @@ async def _refresh_loop(self) -> None: break except Exception as e: self._error_count += 1 - logger.error( - f"Refresh failed for {self.label_key}: {e} " + logger.error( # noqa: G201 + f"Refresh failed for {self.label_key}: {e} " # noqa: G004 f"(error_count={self._error_count})", exc_info=True, ) diff --git a/methods/EverCore/src/core/observation/metrics/server.py b/methods/EverCore/src/core/observation/metrics/server.py index a8516bd3..252838a9 100644 --- a/methods/EverCore/src/core/observation/metrics/server.py +++ b/methods/EverCore/src/core/observation/metrics/server.py @@ -67,11 +67,11 @@ def start_metrics_server(port: Optional[int] = None, addr: str = "0.0.0.0") -> b start_http_server(port=port, addr=addr, registry=get_metrics_registry()) _metrics_server_started = True - logger.info(f"✅ Metrics server started on {addr}:{port}/metrics") + logger.info(f"✅ Metrics server started on {addr}:{port}/metrics") # noqa: G004 return True - except Exception as e: - logger.error(f"Failed to start metrics server: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to start metrics server: {e}") # noqa: G004 return False diff --git a/methods/EverCore/src/core/oxm/es/base_repository.py b/methods/EverCore/src/core/oxm/es/base_repository.py index d2ec73ed..acc82972 100644 --- a/methods/EverCore/src/core/oxm/es/base_repository.py +++ b/methods/EverCore/src/core/oxm/es/base_repository.py @@ -96,7 +96,7 @@ async def get_by_id(self, doc_id: str) -> Optional[T]: try: client = await self.get_client() return await self.model.get(id=doc_id, using=client) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to get document by ID [%s]: %s", self.model_name, e) return None @@ -144,7 +144,7 @@ async def delete_by_id(self, doc_id: str, refresh: bool = False) -> bool: ) return True return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to delete document [%s]: %s", self.model_name, e) return False @@ -168,7 +168,7 @@ async def delete(self, document: T, refresh: bool = False) -> bool: getattr(document, 'meta', {}).get('id', 'unknown'), ) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to delete document [%s]: %s", self.model_name, e) return False @@ -269,7 +269,7 @@ async def match_all(self, size: int = 10, from_: int = 0) -> List[T]: documents.append(doc) return documents - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to get all documents [%s]: %s", self.model_name, e) return [] @@ -291,7 +291,7 @@ async def exists_by_id(self, doc_id: str) -> bool: response = await client.exists(index=index_name, id=doc_id) return response - except Exception: + except Exception: # noqa: BLE001 return False # ==================== Index Management ==================== @@ -321,7 +321,7 @@ async def refresh_index(self) -> bool: except (ConnectionError, TimeoutError) as e: logger.error("❌ Manual index refresh failed [%s]: %s", self.model_name, e) return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Manual index refresh failed (unknown error) [%s]: %s", self.model_name, @@ -367,7 +367,7 @@ async def create_index(self) -> bool: alias, ) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Index creation failed [%s]: %s", self.model_name, e) return False @@ -387,7 +387,7 @@ async def delete_index(self) -> bool: "✅ Index deletion succeeded [%s]: %s", self.model_name, index_name ) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Index deletion failed [%s]: %s", self.model_name, e) return False @@ -403,7 +403,7 @@ async def index_exists(self) -> bool: index_name = self.get_index_name() return await client.indices.exists(index=index_name) - except Exception: + except Exception: # noqa: BLE001 return False # ==================== Helper Methods ==================== diff --git a/methods/EverCore/src/core/oxm/es/migration/utils.py b/methods/EverCore/src/core/oxm/es/migration/utils.py index 184aca85..7a680bac 100644 --- a/methods/EverCore/src/core/oxm/es/migration/utils.py +++ b/methods/EverCore/src/core/oxm/es/migration/utils.py @@ -5,7 +5,6 @@ """ import asyncio -import traceback from typing import Type, Any from elasticsearch import NotFoundError, RequestError from elasticsearch.dsl import AsyncDocument @@ -176,9 +175,8 @@ async def wait_for_task_completion(es_connect: Any, task_id: str) -> None: await asyncio.sleep(5) # Check every 5 seconds - except (NotFoundError, RequestError) as e: - traceback.print_exc() - logger.error("Failed to check task status: %s", e) + except (NotFoundError, RequestError): + logger.exception("Failed to check task status") await asyncio.sleep(10) # Wait longer when error occurs diff --git a/methods/EverCore/src/core/oxm/milvus/base_repository.py b/methods/EverCore/src/core/oxm/milvus/base_repository.py index 3522e9e1..fac2227c 100644 --- a/methods/EverCore/src/core/oxm/milvus/base_repository.py +++ b/methods/EverCore/src/core/oxm/milvus/base_repository.py @@ -87,7 +87,7 @@ async def get_by_id(self, entity_id: str) -> Optional[T]: limit=1, ) return results[0] if results else None - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to get entity by ID [%s]: %s", self.model_name, e) return None @@ -136,7 +136,7 @@ async def delete_by_id(self, entity_id: str, flush: bool = False) -> bool: "✅ Delete entity successful [%s]: %s", self.model_name, entity_id ) return success - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Delete entity failed [%s]: %s", self.model_name, e) return False @@ -182,7 +182,7 @@ async def flush(self) -> bool: await self.collection.flush() logger.debug("✅ Flush collection successful [%s]", self.model_name) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Flush collection failed [%s]: %s", self.model_name, e) return False @@ -197,7 +197,7 @@ async def load(self) -> bool: await self.collection.load() logger.debug("✅ Load collection successful [%s]", self.model_name) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Load collection failed [%s]: %s", self.model_name, e) return False diff --git a/methods/EverCore/src/core/oxm/milvus/migration/utils.py b/methods/EverCore/src/core/oxm/milvus/migration/utils.py index 6d4a4da0..d483e8e8 100644 --- a/methods/EverCore/src/core/oxm/milvus/migration/utils.py +++ b/methods/EverCore/src/core/oxm/milvus/migration/utils.py @@ -70,7 +70,7 @@ def find_collection_manager_by_alias(alias: str) -> Type[MilvusCollectionBase]: base_name = doc_class._COLLECTION_NAME # pylint: disable=protected-access if alias.startswith(base_name): return doc_class - except Exception: # pylint: disable=broad-except # Ignore instantiation failure, continue to next class + except Exception: # pylint: disable=broad-except # Ignore instantiation failure, continue to next class # noqa: BLE001 continue else: # For MilvusCollectionBase, directly compare _COLLECTION_NAME diff --git a/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py b/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py index c590bd54..5c7e834a 100644 --- a/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py +++ b/methods/EverCore/src/core/oxm/milvus/milvus_collection_base.py @@ -190,7 +190,7 @@ def ensure_connection_registered(self) -> None: try: connections._fetch_handler(using) return - except Exception: + except Exception: # noqa: BLE001 pass factory = get_bean("milvus_client_factory") @@ -273,7 +273,7 @@ def _get_existing_indexes(coll: Collection) -> Dict[str, Dict[str, Any]]: return result - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # noqa: BLE001 logger.warning("Error occurred while retrieving index information: %s", e) return {} @@ -508,7 +508,7 @@ def ensure_all(self) -> None: self.name, real_collection_name, ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("Failed to retrieve real collection name: %s", e) logger.info("Collection '%s' initialization completed", self.name) @@ -550,7 +550,7 @@ def create_new_collection(self) -> Collection: try: self._create_indexes_for_collection(new_coll) new_coll.load() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Error occurred while creating indexes for new collection, can be ignored: %s", e, @@ -575,7 +575,7 @@ def switch_alias(self, new_collection: Collection, drop_old: bool = False) -> No old_real_name = ( desc.get("collection_name") if isinstance(desc, dict) else None ) - except Exception: + except Exception: # noqa: BLE001 old_real_name = None # Alias switching @@ -583,11 +583,11 @@ def switch_alias(self, new_collection: Collection, drop_old: bool = False) -> No conn = connections._fetch_handler(self._using) conn.alter_alias(new_real_name, alias_name) logger.info("Alias '%s' switched to '%s'", alias_name, new_real_name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("alter_alias failed, attempting drop/create: %s", e) try: utility.drop_alias(alias_name, using=self._using) - except Exception: + except Exception: # noqa: BLE001 pass utility.create_alias( collection_name=new_real_name, alias=alias_name, using=self._using @@ -599,7 +599,7 @@ def switch_alias(self, new_collection: Collection, drop_old: bool = False) -> No try: utility.drop_collection(old_real_name, using=self._using) logger.info("Deleted old collection: %s", old_real_name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to delete old collection (can be handled manually): %s", e ) @@ -609,7 +609,7 @@ def switch_alias(self, new_collection: Collection, drop_old: bool = False) -> No self.__class__._collection_instance = Collection( name=alias_name, using=self._using ) - except Exception: + except Exception: # noqa: BLE001 pass def exists(self) -> bool: @@ -634,7 +634,7 @@ def drop(self) -> None: utility.drop_collection(real_name, using=self._using) logger.info("Deleted Collection '%s'", real_name) - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # noqa: BLE001 logger.warning( "Collection '%s' does not exist or deletion failed: %s", self.name, e ) diff --git a/methods/EverCore/src/core/oxm/mongo/base_repository.py b/methods/EverCore/src/core/oxm/mongo/base_repository.py index 641f47fe..47f7f9b7 100644 --- a/methods/EverCore/src/core/oxm/mongo/base_repository.py +++ b/methods/EverCore/src/core/oxm/mongo/base_repository.py @@ -138,7 +138,7 @@ async def get_by_id(self, object_id: Union[str, PydanticObjectId]) -> Optional[T if isinstance(object_id, str): object_id = PydanticObjectId(object_id) return await self.model.get(object_id) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to get document by ID [%s]: %s", self.model_name, e) return None @@ -193,7 +193,7 @@ async def delete_by_id( ) return True return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to delete document [%s]: %s", self.model_name, e) return False @@ -218,7 +218,7 @@ async def delete( getattr(document, 'id', 'unknown'), ) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to delete document [%s]: %s", self.model_name, e) return False @@ -331,7 +331,7 @@ async def count_all(self, filter_query: Optional[dict] = None) -> int: count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count documents [%s]: %s", self.model_name, e) return 0 @@ -350,7 +350,7 @@ async def exists_by_id(self, object_id: Union[str, PydanticObjectId]) -> bool: object_id = PydanticObjectId(object_id) document = await self.model.get(object_id) return document is not None - except Exception: + except Exception: # noqa: BLE001 return False # ==================== Helper Methods ==================== diff --git a/methods/EverCore/src/core/oxm/mongo/migration/cli.py b/methods/EverCore/src/core/oxm/mongo/migration/cli.py index bcf9ff37..6784741b 100644 --- a/methods/EverCore/src/core/oxm/mongo/migration/cli.py +++ b/methods/EverCore/src/core/oxm/mongo/migration/cli.py @@ -118,10 +118,10 @@ def main(): try: filepath = manager.create_migration(args.name) logger.info("🎉 Migration file created successfully!") - logger.info(f"📝 Please edit file: {filepath}") + logger.info(f"📝 Please edit file: {filepath}") # noqa: G004 - except Exception as e: - logger.error(f"❌ Failed to create migration: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"❌ Failed to create migration: {e}") # noqa: G004 sys.exit(1) elif args.command == "migrate": @@ -135,12 +135,12 @@ def main(): sys.exit(exit_code) else: - logger.error(f"❌ Unknown command: {sys.argv[1]}") + logger.error(f"❌ Unknown command: {sys.argv[1]}") # noqa: G004 show_help() sys.exit(1) - except Exception as e: - logger.error(f"❌ Error: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"❌ Error: {e}") # noqa: G004 sys.exit(1) diff --git a/methods/EverCore/src/core/oxm/mongo/migration/manager.py b/methods/EverCore/src/core/oxm/mongo/migration/manager.py index fd0c6062..f1f52e6f 100644 --- a/methods/EverCore/src/core/oxm/mongo/migration/manager.py +++ b/methods/EverCore/src/core/oxm/mongo/migration/manager.py @@ -178,7 +178,7 @@ def create_migration(self, migration_name: str) -> Path: # Write file filepath.write_text(content, encoding='utf-8') - logger.info(f"✅ Created migration file: {filepath}") + logger.info(f"✅ Created migration file: {filepath}") # noqa: G004 return filepath @@ -212,9 +212,9 @@ def run_migration(self) -> int: str(self.migrations_path), ] - logger.info(f"🚀 Executing command: {' '.join(cmd[3:])}") # Hide python path - logger.info(f"📍 Database: {self.database}") - logger.info(f"📁 Migration directory: {self.migrations_path}") + logger.info(f"🚀 Executing command: {' '.join(cmd[3:])}") # Hide python path # noqa: G004 + logger.info(f"📍 Database: {self.database}") # noqa: G004 + logger.info(f"📁 Migration directory: {self.migrations_path}") # noqa: G004 # Check if there are migration files in the directory migration_files = list(self.migrations_path.glob("*.py")) @@ -222,14 +222,14 @@ def run_migration(self) -> int: if not migration_files: logger.info("🧭 No migration files found in directory, skipping migration") return 0 - logger.info(f"📄 Found {len(migration_files)} migration files") + logger.info(f"📄 Found {len(migration_files)} migration files") # noqa: G004 # Snapshot migration logs before running before_names, before_current = self._snapshot_migration_log() if before_names is not None: - logger.info(f"🧭 Number of records before migration: {len(before_names)}") + logger.info(f"🧭 Number of records before migration: {len(before_names)}") # noqa: G004 logger.info( - f"⭐ Current pointer before migration: {before_current or ''}" + f"⭐ Current pointer before migration: {before_current or ''}" # noqa: G004 ) else: logger.info( @@ -268,11 +268,11 @@ def run_migration(self) -> int: return result.returncode except subprocess.CalledProcessError as e: - logger.error(f"❌ Command execution failed: {e}") + logger.error(f"❌ Command execution failed: {e}") # noqa: G004 if e.stdout: - logger.info(f"Standard output: {e.stdout}") + logger.info(f"Standard output: {e.stdout}") # noqa: G004 if e.stderr: - logger.error(f"Error output: {e.stderr}") + logger.error(f"Error output: {e.stderr}") # noqa: G004 # Snapshot and log diff even on failure (migration may have partially executed) self._log_migration_diff(before_names, before_current) return e.returncode @@ -313,7 +313,7 @@ def _read_migration_logs(self): current = d.get("name") break return names, current - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("Failed to read migration logs: %s", str(e)) return None, None @@ -373,7 +373,7 @@ def get_migration_history(self): ) ) return docs - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("Failed to get migration history: %s", str(e)) return [] @@ -434,6 +434,6 @@ def run_migrations_on_startup(cls, enabled: bool = True) -> int: return exit_code - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Error during MongoDB migration: %s", str(e)) return 1 diff --git a/methods/EverCore/src/core/oxm/pg/audit_base.py b/methods/EverCore/src/core/oxm/pg/audit_base.py index 159bcc69..fcdc0f99 100644 --- a/methods/EverCore/src/core/oxm/pg/audit_base.py +++ b/methods/EverCore/src/core/oxm/pg/audit_base.py @@ -136,6 +136,6 @@ def _get_current_user_id() -> Optional[str]: user_info = get_current_user_info() if user_info and 'user_id' in user_info: return str(user_info['user_id']) - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # noqa: BLE001 logger.debug("Failed to get current user information: %s", e) return None diff --git a/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py b/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py index ddca40cf..4ca41c34 100644 --- a/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py +++ b/methods/EverCore/src/core/queue/redis_group_queue/kafka_consumer_record_item.py @@ -183,7 +183,7 @@ def to_consumer_record(self) -> ConsumerRecord: if isinstance(key, str): try: key = self._decode_base64_to_bytes(key) - except Exception: + except Exception: # noqa: BLE001 # If decoding fails, keep original string pass @@ -194,7 +194,7 @@ def to_consumer_record(self) -> ConsumerRecord: try: # Try to decode from base64 headers_bytes.append((name, self._decode_base64_to_bytes(data))) - except Exception: + except Exception: # noqa: BLE001 # If decoding fails, encode as UTF-8 bytes headers_bytes.append((name, data.encode('utf-8'))) else: diff --git a/methods/EverCore/src/core/request/app_logic_provider.py b/methods/EverCore/src/core/request/app_logic_provider.py index 7dfe4822..eb99a378 100644 --- a/methods/EverCore/src/core/request/app_logic_provider.py +++ b/methods/EverCore/src/core/request/app_logic_provider.py @@ -185,7 +185,7 @@ def _init_token_usage_collector(self) -> None: collector = get_bean_by_type(TokenUsageCollector) collector.reset() - except Exception: + except Exception: # noqa: BLE001 pass def _cleanup_token_usage_collector(self) -> None: @@ -199,7 +199,7 @@ def _cleanup_token_usage_collector(self) -> None: collector = get_bean_by_type(TokenUsageCollector) collector.reset() - except Exception: + except Exception: # noqa: BLE001 pass diff --git a/methods/EverCore/src/core/request/request_history_config.py b/methods/EverCore/src/core/request/request_history_config.py index 4b2f44d7..87365a13 100644 --- a/methods/EverCore/src/core/request/request_history_config.py +++ b/methods/EverCore/src/core/request/request_history_config.py @@ -115,7 +115,7 @@ def is_request_history_enabled() -> bool: _enabled_cache = config.is_enabled() logger.info( - f"Request history logging is {'enabled' if _enabled_cache else 'disabled'} " + f"Request history logging is {'enabled' if _enabled_cache else 'disabled'} " # noqa: G004 f"(config: {config.get_config_name()})" ) diff --git a/methods/EverCore/src/core/request/request_history_decorator.py b/methods/EverCore/src/core/request/request_history_decorator.py index 7070172d..74e1504d 100644 --- a/methods/EverCore/src/core/request/request_history_decorator.py +++ b/methods/EverCore/src/core/request/request_history_decorator.py @@ -44,8 +44,8 @@ async def _extract_request_body(request: Request) -> Optional[str]: body_bytes = await request.body() if body_bytes: return body_bytes.decode("utf-8", errors="replace") - except Exception as e: - logger.debug(f"Failed to extract request body: {e}") + except Exception as e: # noqa: BLE001 + logger.debug(f"Failed to extract request body: {e}") # noqa: G004 return None @@ -102,11 +102,11 @@ async def _publish_request_history_event(event: RequestHistoryEvent) -> None: publisher = get_bean_by_type(ApplicationEventPublisher) await publisher.publish(event) logger.debug( - f"Published request history event: {event.method} " + f"Published request history event: {event.method} " # noqa: G004 f"endpoint={event.endpoint_name} version={event.version}" ) - except Exception as e: - logger.warning(f"Failed to publish request history event: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to publish request history event: {e}") # noqa: G004 def log_request( @@ -198,8 +198,8 @@ async def async_wrapper(*args: Any, **kwargs: Any) -> Any: else: await _publish_request_history_event(event) - except Exception as e: - logger.warning(f"Failed to log request: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to log request: {e}") # noqa: G004 # Call the original function return await func(*args, **kwargs) @@ -240,11 +240,11 @@ def sync_wrapper(*args: Any, **kwargs: Any) -> Any: try: publisher = get_bean_by_type(ApplicationEventPublisher) publisher.publish_sync(event) - except Exception as e: - logger.warning(f"Failed to publish request history event: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to publish request history event: {e}") # noqa: G004 - except Exception as e: - logger.warning(f"Failed to log request: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to log request: {e}") # noqa: G004 return func(*args, **kwargs) diff --git a/methods/EverCore/src/core/request/timeout_background.py b/methods/EverCore/src/core/request/timeout_background.py index 80e6c4b0..bf0ef81d 100644 --- a/methods/EverCore/src/core/request/timeout_background.py +++ b/methods/EverCore/src/core/request/timeout_background.py @@ -16,7 +16,6 @@ from typing import Any, Callable, Coroutine, TypeVar, ParamSpec, Union, Optional from functools import wraps import asyncio -import traceback from fastapi import Request from fastapi.responses import JSONResponse @@ -193,12 +192,9 @@ async def _run_background_task( "[TimeoutBackground] Background task '%s' was cancelled", task_name ) except Exception as e: - logger.error( - "[TimeoutBackground] Background task '%s' execution failed: %s", - task_name, - e, + logger.exception( + "[TimeoutBackground] Background task '%s' execution failed", task_name ) - traceback.print_exc() await _call_on_request_complete(provider, http_code=500, error_message=str(e)) @@ -222,7 +218,7 @@ async def _call_on_request_complete( await provider.on_request_complete( request=request, http_code=http_code, error_message=error_message ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "[TimeoutBackground] on_request_complete callback execution failed: %s", e ) diff --git a/methods/EverCore/src/core/tenants/init_tenant_all.py b/methods/EverCore/src/core/tenants/init_tenant_all.py index 98012333..dce58106 100644 --- a/methods/EverCore/src/core/tenants/init_tenant_all.py +++ b/methods/EverCore/src/core/tenants/init_tenant_all.py @@ -127,7 +127,7 @@ async def init_mongodb() -> bool: logger.info("✅ Tenant's MongoDB database initialized successfully") await mongodb_provider.shutdown(mock_app) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to initialize tenant's MongoDB database: %s", e) return False @@ -145,7 +145,7 @@ async def init_milvus() -> bool: logger.info("✅ Tenant's Milvus database initialized successfully") await milvus_provider.shutdown(mock_app) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to initialize tenant's Milvus database: %s", e) return False @@ -163,7 +163,7 @@ async def init_elasticsearch() -> bool: logger.info("✅ Tenant's Elasticsearch database initialized successfully") await es_provider.shutdown(mock_app) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to initialize tenant's Elasticsearch database: %s", e) return False diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/es/config_utils.py b/methods/EverCore/src/core/tenants/tenantize/oxm/es/config_utils.py index 52f22bf4..ba3e45ce 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/es/config_utils.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/es/config_utils.py @@ -243,7 +243,7 @@ def get_tenant_aware_index_name(original_name: str) -> str: ) return _base_prefixed_index_name(original_name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to get tenant-aware index name, using base prefix: %s", e ) diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_aware_async_document.py b/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_aware_async_document.py index d86c4de4..d2d2224b 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_aware_async_document.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/es/tenant_aware_async_document.py @@ -156,7 +156,7 @@ def _ensure_connection_registered(cls, using: str) -> None: async_connections.get_connection(using) # Connection exists, return directly return - except Exception: + except Exception: # noqa: BLE001 # Connection does not exist, need to register pass diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/config_utils.py b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/config_utils.py index 050183cf..0a857957 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/config_utils.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/config_utils.py @@ -201,7 +201,7 @@ def get_tenant_aware_collection_name(original_name: str) -> str: ) return _base_prefixed_collection_name(original_name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to get tenant-aware Collection name, using base prefix: %s", e ) diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection.py b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection.py index f61baf61..d62275f0 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection.py @@ -161,7 +161,7 @@ def _ensure_connection_registered(using: str) -> None: connections._fetch_handler(using) # Connection exists, return directly return - except Exception: + except Exception: # noqa: BLE001 # Connection does not exist, needs registration pass diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py index a932feb9..f5e4804a 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/milvus/tenant_aware_collection_with_suffix.py @@ -288,7 +288,7 @@ def load_collection(self) -> TenantAwareCollection: # Note: First delete any existing old alias try: utility.drop_alias(tenant_aware_alias_name, using=using) - except Exception: + except Exception: # noqa: BLE001 pass # alias does not exist, ignore utility.create_alias( @@ -425,7 +425,7 @@ def switch_alias( old_real_name = ( desc.get("collection_name") if isinstance(desc, dict) else None ) - except Exception: + except Exception: # noqa: BLE001 old_real_name = None # Alias switching @@ -437,11 +437,11 @@ def switch_alias( tenant_aware_alias_name, tenant_aware_new_real_name, ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("alter_alias failed, trying drop/create: %s", e) try: utility.drop_alias(tenant_aware_alias_name, using=using) - except Exception: + except Exception: # noqa: BLE001 pass utility.create_alias( collection_name=tenant_aware_new_real_name, @@ -459,7 +459,7 @@ def switch_alias( try: utility.drop_collection(old_real_name, using=using) logger.info("Old collection deleted: %s", old_real_name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to delete old collection (can handle manually): %s", e ) @@ -471,7 +471,7 @@ def switch_alias( schema=self._SCHEMA, consistency_level=ConsistencyLevel.Bounded, ) - except Exception: + except Exception: # noqa: BLE001 pass # ==================== Tenant Field Isolation ==================== @@ -538,7 +538,7 @@ def drop(self) -> None: try: utility.drop_collection(name, using=using) logger.info("Collection '%s' deleted", name) - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except # noqa: BLE001 logger.warning( "Collection '%s' does not exist or deletion failed: %s", name, e ) diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/config_utils.py b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/config_utils.py index f0a9771e..2285bc53 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/config_utils.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/config_utils.py @@ -231,6 +231,6 @@ def generate_tenant_database_name(base_name: str = "memsys") -> str: ) return _base_prefixed_database_name(base_name) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("Failed to get tenant database name, using base prefix: %s", e) return _base_prefixed_database_name(base_name) diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_client_factory.py b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_client_factory.py index 64933f27..61cd25f4 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_client_factory.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_client_factory.py @@ -225,7 +225,7 @@ async def test_connection(self) -> bool: await real_client.admin.command('ping') logger.info("✅ MongoDB connection test succeeded (tenant-aware)") return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ MongoDB connection test failed (tenant-aware): %s", e) return False diff --git a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py index e06df552..6d3606ba 100644 --- a/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py +++ b/methods/EverCore/src/core/tenants/tenantize/oxm/mongo/tenant_aware_mongo_client.py @@ -352,7 +352,7 @@ async def close(self): try: await client.close() logger.info("🔌 MongoDB client closed [cache_key=%s]", cache_key) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Failed to close client [cache_key=%s]: %s", cache_key, e ) @@ -364,7 +364,7 @@ async def close(self): try: await self._fallback_client.close() logger.info("🔌 Fallback MongoDB client closed") - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to close fallback client: %s", e) self._fallback_client = None diff --git a/methods/EverCore/src/core/tenants/tenantize/tenant_cache_utils.py b/methods/EverCore/src/core/tenants/tenantize/tenant_cache_utils.py index 88a49c15..c83ffce1 100644 --- a/methods/EverCore/src/core/tenants/tenantize/tenant_cache_utils.py +++ b/methods/EverCore/src/core/tenants/tenantize/tenant_cache_utils.py @@ -191,7 +191,7 @@ def _resolve_fallback( if callable(fallback): try: return fallback() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to compute fallback %s: %s", description, e) return None diff --git a/methods/EverCore/src/devops_scripts/data_fix/es_rebuild_index.py b/methods/EverCore/src/devops_scripts/data_fix/es_rebuild_index.py index 83bd3677..412f0fd0 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/es_rebuild_index.py +++ b/methods/EverCore/src/devops_scripts/data_fix/es_rebuild_index.py @@ -1,6 +1,5 @@ import argparse import asyncio -import traceback from elasticsearch.dsl import AsyncDocument @@ -24,9 +23,8 @@ async def run(index_name: str, close_old: bool, delete_old: bool) -> None: logger.info("Index alias: %s", document_class.get_index_name()) await rebuild_index(document_class, close_old=close_old, delete_old=delete_old) - except Exception as exc: # noqa: BLE001 - logger.error("Failed to rebuild index: %s", exc) - traceback.print_exc() + except Exception: # noqa: BLE001 + logger.exception("Failed to rebuild index") raise diff --git a/methods/EverCore/src/devops_scripts/data_fix/es_sync_docs.py b/methods/EverCore/src/devops_scripts/data_fix/es_sync_docs.py index fa420f8a..1abd351a 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/es_sync_docs.py +++ b/methods/EverCore/src/devops_scripts/data_fix/es_sync_docs.py @@ -1,6 +1,5 @@ import argparse import asyncio -import traceback from elasticsearch.dsl import AsyncDocument from core.observation.logger import get_logger @@ -37,9 +36,8 @@ async def run( ) else: raise ValueError(f"Unsupported index type: {doc_alias}") - except Exception as exc: # noqa: BLE001 - logger.error("Failed to synchronize documents: %s", exc) - traceback.print_exc() + except Exception: # noqa: BLE001 + logger.exception("Failed to synchronize documents") raise diff --git a/methods/EverCore/src/devops_scripts/data_fix/es_sync_episodic_memory_docs.py b/methods/EverCore/src/devops_scripts/data_fix/es_sync_episodic_memory_docs.py index 057f99b6..1c0b3cf2 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/es_sync_episodic_memory_docs.py +++ b/methods/EverCore/src/devops_scripts/data_fix/es_sync_episodic_memory_docs.py @@ -1,4 +1,3 @@ -import traceback from datetime import timedelta from typing import Optional, AsyncIterator, Dict, Any @@ -141,7 +140,6 @@ async def generate_actions() -> AsyncIterator[Dict[str, Any]]: success_count, error_count, ) - except Exception as exc: # noqa: BLE001 - logger.error("An error occurred during sync: %s", exc) - traceback.print_exc() + except Exception: # noqa: BLE001 + logger.exception("An error occurred during sync") raise diff --git a/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py b/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py index 816d8b82..2d81c187 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py +++ b/methods/EverCore/src/devops_scripts/data_fix/milvus_rebuild_collection.py @@ -21,7 +21,6 @@ import argparse import sys -import traceback from typing import Optional, List from pymilvus import Collection @@ -269,8 +268,7 @@ def populate_fn(old_col, new_col): "error": str(exc)[:500], } ) - logger.error("Milvus rebuild failed: %s", exc) - traceback.print_exc() + logger.exception("Milvus rebuild failed") raise @@ -398,7 +396,7 @@ def main(argv: Optional[List[str]] = None) -> int: progress=progress, ) logger.info("[OK] %s", alias) - except Exception: + except Exception: # noqa: BLE001 logger.error("[FAIL] %s", alias) failed += 1 diff --git a/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_docs.py b/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_docs.py index d42f2aaa..b17739ee 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_docs.py +++ b/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_docs.py @@ -16,7 +16,6 @@ import argparse import asyncio -import traceback from core.observation.logger import get_logger @@ -57,9 +56,8 @@ async def run( else: raise ValueError(f"Unsupported Collection type: {collection_name}") - except Exception as exc: # noqa: BLE001 - logger.error("Failed to sync documents: %s", exc) - traceback.print_exc() + except Exception: # noqa: BLE001 + logger.exception("Failed to sync documents") raise diff --git a/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_episodic_memory_docs.py b/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_episodic_memory_docs.py index 91e1d2cc..fe098d9e 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_episodic_memory_docs.py +++ b/methods/EverCore/src/devops_scripts/data_fix/milvus_sync_episodic_memory_docs.py @@ -12,7 +12,6 @@ - Supports idempotent operations (using upsert semantics) """ -import traceback from datetime import timedelta from typing import Optional, List, Dict, Any @@ -164,9 +163,8 @@ async def sync_episodic_memory_docs( success_count += inserted_count logger.info("Bulk insert successful: %d records", inserted_count) - except Exception as e: # noqa: BLE001 - logger.error("Bulk insert to Milvus failed: %s", e) - traceback.print_exc() + except Exception: # noqa: BLE001 + logger.exception("Bulk insert to Milvus failed") error_count += len(milvus_entities) # Update statistics @@ -199,7 +197,6 @@ async def sync_episodic_memory_docs( error_count, ) - except Exception as exc: # noqa: BLE001 - logger.error("Error occurred during sync: %s", exc) - traceback.print_exc() + except Exception: # noqa: BLE001 + logger.exception("Error occurred during sync") raise diff --git a/methods/EverCore/src/devops_scripts/data_fix/mongo_add_timestamp_shard.py b/methods/EverCore/src/devops_scripts/data_fix/mongo_add_timestamp_shard.py index 1f26d622..f9c8aa47 100644 --- a/methods/EverCore/src/devops_scripts/data_fix/mongo_add_timestamp_shard.py +++ b/methods/EverCore/src/devops_scripts/data_fix/mongo_add_timestamp_shard.py @@ -39,23 +39,23 @@ async def enable_timestamp_sharding(session=None): ) return logger.info( - f"✅ Sharded cluster detected, total {len(shard_status['shards'])} shards" + f"✅ Sharded cluster detected, total {len(shard_status['shards'])} shards" # noqa: G004 ) except OperationFailure as e: logger.warning( - f"⚠️ Unable to check sharding status: {e}, may not be a sharded environment" + f"⚠️ Unable to check sharding status: {e}, may not be a sharded environment" # noqa: G004 ) return # 2. Enable database sharding try: await admin_db.command('enableSharding', db.name) - logger.info(f"✅ Sharding enabled for database '{db.name}'") + logger.info(f"✅ Sharding enabled for database '{db.name}'") # noqa: G004 except OperationFailure as e: if "already enabled" in str(e).lower(): - logger.info(f"📝 Sharding already exists for database '{db.name}'") + logger.info(f"📝 Sharding already exists for database '{db.name}'") # noqa: G004 else: - logger.error(f"❌ Failed to enable database sharding: {e}") + logger.error(f"❌ Failed to enable database sharding: {e}") # noqa: G004 raise # 3. Set collection shard key - timestamp @@ -69,7 +69,7 @@ async def enable_timestamp_sharding(session=None): if "already sharded" in str(e).lower(): logger.info("📝 Sharding already exists for MemCell collection") else: - logger.error(f"❌ Failed to set collection sharding: {e}") + logger.error(f"❌ Failed to set collection sharding: {e}") # noqa: G004 raise # 4. Create pre-split chunks (optional, improves initial performance) @@ -90,15 +90,15 @@ async def enable_timestamp_sharding(session=None): for point in split_points: try: await admin_db.command('split', collection_name, middle=point) - logger.debug(f"📅 Created split point: {point['timestamp']}") + logger.debug(f"📅 Created split point: {point['timestamp']}") # noqa: G004 except OperationFailure as e: if "already exists" not in str(e).lower(): - logger.debug(f"Failed to create pre-split point: {e}") + logger.debug(f"Failed to create pre-split point: {e}") # noqa: G004 - logger.info(f"✅ Created {len(split_points)} pre-split points") + logger.info(f"✅ Created {len(split_points)} pre-split points") # noqa: G004 - except Exception as e: - logger.warning(f"⚠️ Pre-splitting creation failed: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"⚠️ Pre-splitting creation failed: {e}") # noqa: G004 # 5. Verify sharding configuration try: @@ -108,17 +108,17 @@ async def enable_timestamp_sharding(session=None): logger.info( "✅ MemCell collection sharding configuration verified successfully" ) - logger.info(f"📊 Shard key: {shard_info.get('shardKey', {})}") + logger.info(f"📊 Shard key: {shard_info.get('shardKey', {})}") # noqa: G004 else: logger.warning("⚠️ Sharding configuration verification failed") - except Exception as e: - logger.warning(f"⚠️ Sharding verification failed: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"⚠️ Sharding verification failed: {e}") # noqa: G004 logger.info("🎉 Timestamp sharding configuration completed") except Exception as e: - logger.error(f"❌ Error occurred during sharding configuration: {e}") + logger.error(f"❌ Error occurred during sharding configuration: {e}") # noqa: G004 raise diff --git a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py index ba3eddd8..db38c644 100644 --- a/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py +++ b/methods/EverCore/src/devops_scripts/i18n/i18n_tool.py @@ -440,7 +440,7 @@ def load_translation_progress() -> dict: try: with open(TRANSLATION_PROGRESS_FILE, 'r', encoding='utf-8') as f: return json.load(f) - except Exception: + except Exception: # noqa: BLE001 return {"processed": [], "errors": []} return {"processed": [], "errors": []} @@ -468,7 +468,7 @@ def load_review_progress() -> dict: try: with open(REVIEW_PROGRESS_FILE, 'r', encoding='utf-8') as f: return json.load(f) - except Exception: + except Exception: # noqa: BLE001 return {"commit_range": "", "safe": [], "needs_review": [], "errors": []} return {"commit_range": "", "safe": [], "needs_review": [], "errors": []} @@ -506,7 +506,7 @@ def run_git_command(args: list[str], cwd: Path | None = None) -> tuple[bool, str return False, result.stderr.strip() except subprocess.TimeoutExpired: return False, "Git command timed out" - except Exception as e: + except Exception as e: # noqa: BLE001 return False, str(e) @@ -618,7 +618,7 @@ def filter_files_with_chinese( progress["processed"].append(file_str) else: skipped_already_done += 1 - except Exception as e: + except Exception as e: # noqa: BLE001 print(f" Warning: Could not pre-scan {file_path}: {e}") files_to_process.append(file_path) @@ -710,7 +710,7 @@ async def translate_file( save_translation_progress(progress) return (file_path, True, None) - except Exception as e: + except Exception as e: # noqa: BLE001 error_msg = str(e) print(f"{progress_prefix} [ERROR] {file_path}: {error_msg}") async with progress_lock: @@ -820,7 +820,7 @@ async def review_file_diff( return result - except Exception as e: + except Exception as e: # noqa: BLE001 result = FileReviewResult( file_path=file_path, result=ReviewResult.ERROR, @@ -995,7 +995,7 @@ def cmd_check(directories: list[Path], specific_files: list[str] | None = None) } ) - except Exception as e: + except Exception as e: # noqa: BLE001 print(f" [ERROR] Could not read {file_path}: {e}") print("=" * 70) @@ -1410,7 +1410,7 @@ def _hook_check_files( if cjk_lines: files_with_cjk[file_path] = cjk_lines - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"Warning: Could not read {file_path}: {e}", file=sys.stderr) return bool(files_with_cjk), files_with_cjk @@ -1435,7 +1435,7 @@ def _hook_check_commit_message(msg_file: str) -> tuple[bool, list[tuple[int, str cjk_lines = _hook_find_cjk_lines(content) return bool(cjk_lines), cjk_lines - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"Warning: Could not read commit message file: {e}", file=sys.stderr) return False, [] diff --git a/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py b/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py index 1233add3..a8b1d9bc 100644 --- a/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py +++ b/methods/EverCore/src/devops_scripts/milvus_admin/browse_collections.py @@ -88,13 +88,13 @@ def show_collection_details(client: MilvusClient, names: List[str]) -> None: try: stats = client.get_collection_stats(name) row_count = int(stats.get("row_count", 0)) - except Exception: + except Exception: # noqa: BLE001 row_count = -1 try: aliases = utility.list_aliases(collection_name=name, using="_admin") alias_str = ", ".join(aliases) if aliases else "-" - except Exception: + except Exception: # noqa: BLE001 alias_str = "?" row_str = _format_row_count(row_count) if row_count >= 0 else "?" @@ -177,13 +177,13 @@ def drop_collection(client: MilvusClient, name: str) -> bool: for alias in aliases: utility.drop_alias(alias, using="_admin") print(f" Dropped alias: {alias}") - except Exception: + except Exception: # noqa: BLE001 pass client.drop_collection(name) print(f" Deleted: {name}") return True - except Exception as e: + except Exception as e: # noqa: BLE001 print(f" Failed to delete {name}: {e}") return False @@ -283,7 +283,7 @@ def main(argv: Optional[List[str]] = None) -> int: try: client.close() connections.disconnect("_admin") - except Exception: + except Exception: # noqa: BLE001 pass return 0 diff --git a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py index 0f2d505f..20c96543 100644 --- a/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py +++ b/methods/EverCore/src/devops_scripts/sensitive_info/sensitive_info_tool.py @@ -479,7 +479,7 @@ async def analyze_file_with_ai( file_path, "r", encoding="utf-8", errors="ignore" ) as f: content = await f.read() - except Exception as e: + except Exception as e: # noqa: BLE001 return FileCheckResult(file_path=file_path, error=str(e)) # Skip empty files @@ -538,7 +538,7 @@ async def analyze_file_with_ai( ai_analysis=response[:500] if len(response) > 500 else response, ) - except Exception as e: + except Exception as e: # noqa: BLE001 print(f"{progress_prefix} [ERROR] {rel_path}: {e}") return FileCheckResult(file_path=file_path, error=str(e)) @@ -884,7 +884,7 @@ def cmd_hook(files: list[str], commit_msg: bool = False) -> int: for keyword in SKIP_COMMIT_MSG_KEYWORDS: if keyword.lower() in content: return 0 - except Exception: + except Exception: # noqa: BLE001 pass return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/memory/group_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/memory/group_controller.py index eed2693c..d1100648 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/memory/group_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/memory/group_controller.py @@ -101,7 +101,7 @@ async def create_group( except HTTPException: raise except Exception as e: - logger.error("Group create failed: %s", e, exc_info=True) + logger.error("Group create failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException(status_code=500, detail="Failed to create group") from e @get( @@ -146,7 +146,7 @@ async def get_group( except HTTPException: raise except Exception as e: - logger.error("Group get failed: %s", e, exc_info=True) + logger.error("Group get failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to retrieve group" ) from e @@ -224,5 +224,5 @@ async def patch_group( except HTTPException: raise except Exception as e: - logger.error("Group patch failed: %s", e, exc_info=True) + logger.error("Group patch failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException(status_code=500, detail="Failed to update group") from e diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py index 640ac910..07ed6afa 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_controller.py @@ -209,7 +209,7 @@ async def add_personal_memories( ) raise except Exception as e: - logger.error("Personal add failed: %s", e, exc_info=True) + logger.error("Personal add failed: %s", e, exc_info=True) # noqa: G201 error_type = classify_memorize_error(e) record_memorize_error( space_id=space_id, @@ -356,7 +356,7 @@ async def add_group_memories( ) raise except Exception as e: - logger.error("Group add failed: %s", e, exc_info=True) + logger.error("Group add failed: %s", e, exc_info=True) # noqa: G201 error_type = classify_memorize_error(e) record_memorize_error( space_id=space_id, @@ -425,7 +425,7 @@ async def flush_personal_memories( logger.error("Personal flush parameter error: %s", e) raise HTTPException(status_code=422, detail=str(e)) from e except Exception as e: - logger.error("Personal flush failed: %s", e, exc_info=True) + logger.error("Personal flush failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Flush failed, please try again later" ) from e @@ -477,7 +477,7 @@ async def flush_group_memories( logger.error("Group flush parameter error: %s", e) raise HTTPException(status_code=422, detail=str(e)) from e except Exception as e: - logger.error("Group flush failed: %s", e, exc_info=True) + logger.error("Group flush failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Flush failed, please try again later" ) from e @@ -598,7 +598,7 @@ async def flush_agent_memories( logger.error("Agent flush parameter error: %s", e) raise HTTPException(status_code=422, detail=str(e)) from e except Exception as e: - logger.error("Agent flush failed: %s", e, exc_info=True) + logger.error("Agent flush failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Flush failed, please try again later" ) from e @@ -866,7 +866,7 @@ async def add_agent_memories( ) raise except Exception as e: - logger.error("Agent add failed: %s", e, exc_info=True) + logger.error("Agent add failed: %s", e, exc_info=True) # noqa: G201 record_memorize_error( space_id=space_id, raw_data_type=raw_data_type, @@ -912,7 +912,7 @@ async def _ensure_group_exists( await group_service.ensure_group_exists( group_id=group_id, name=name, description=description ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to auto-register group: group_id=%s, error=%s", group_id, e ) @@ -950,7 +950,7 @@ async def _enrich_sender_names(self, messages: list, raw_data_list: list) -> Non sid = raw_data.content.get("sender_id") if sid in name_map: raw_data.content["sender_name"] = name_map[sid] - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("Failed to enrich sender names: %s", e) async def _ensure_sender_exists( @@ -964,7 +964,7 @@ async def _ensure_sender_exists( await sender_service.ensure_sender_exists( sender_id=sender_id, name=sender_name ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to auto-register sender: sender_id=%s, error=%s", sender_id, e ) @@ -974,7 +974,7 @@ async def _publish_event(self, event) -> None: try: publisher = get_bean_by_type(ApplicationEventPublisher) await publisher.publish(event) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("Failed to publish event %s: %s", type(event).__name__, e) async def _ensure_session_exists(self, session_id: str) -> None: @@ -984,7 +984,7 @@ async def _ensure_session_exists(self, session_id: str) -> None: session_service = get_bean_by_type(SessionService) await session_service.ensure_session_exists(session_id=session_id) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( "Failed to auto-register session: session_id=%s, error=%s", session_id, diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_get_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_get_controller.py index 6e02f29e..94542053 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_get_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_get_controller.py @@ -175,7 +175,7 @@ async def get_memories( # 1. Parse and validate request body try: body = await fastapi_request.json() - except Exception: + except Exception: # noqa: BLE001 raise HTTPException(status_code=400, detail="Invalid JSON request body") try: @@ -194,7 +194,7 @@ async def get_memories( except InvalidScopeError as e: raise HTTPException(status_code=422, detail=str(e)) from e except Exception as e: - logger.error("get_memories failed: %s", e, exc_info=True) + logger.error("get_memories failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to retrieve memory, please try again later", diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_search_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_search_controller.py index cd9393da..ede7d7e9 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_search_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/memory/memory_search_controller.py @@ -175,7 +175,7 @@ async def search_memories( # 1. Parse and validate request body try: body = await fastapi_request.json() - except Exception: + except Exception: # noqa: BLE001 raise HTTPException(status_code=400, detail="Invalid JSON request body") try: @@ -206,7 +206,7 @@ async def search_memories( logger.error("search_memories validation error: %s", e) raise HTTPException(status_code=422, detail=str(e)) from e except Exception as e: - logger.error("search_memories failed: %s", e, exc_info=True) + logger.error("search_memories failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to search memories, please try again later", diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/memory/sender_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/memory/sender_controller.py index acf6bf6d..8c6df54d 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/memory/sender_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/memory/sender_controller.py @@ -98,7 +98,7 @@ async def create_sender( except HTTPException: raise except Exception as e: - logger.error("Sender create failed: %s", e, exc_info=True) + logger.error("Sender create failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to create sender" ) from e @@ -145,7 +145,7 @@ async def get_sender( except HTTPException: raise except Exception as e: - logger.error("Sender get failed: %s", e, exc_info=True) + logger.error("Sender get failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to retrieve sender" ) from e @@ -204,7 +204,7 @@ async def patch_sender( except HTTPException: raise except Exception as e: - logger.error("Sender patch failed: %s", e, exc_info=True) + logger.error("Sender patch failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to update sender" ) from e diff --git a/methods/EverCore/src/infra_layer/adapters/input/api/memory/settings_controller.py b/methods/EverCore/src/infra_layer/adapters/input/api/memory/settings_controller.py index b5857ac3..22fd8df6 100644 --- a/methods/EverCore/src/infra_layer/adapters/input/api/memory/settings_controller.py +++ b/methods/EverCore/src/infra_layer/adapters/input/api/memory/settings_controller.py @@ -79,7 +79,7 @@ async def get_settings(self, request: FastAPIRequest) -> GetSettingsApiResponse: except HTTPException: raise except Exception as e: - logger.error("Settings get failed: %s", e, exc_info=True) + logger.error("Settings get failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to retrieve settings" ) from e @@ -137,7 +137,7 @@ async def update_settings( except HTTPException: raise except Exception as e: - logger.error("Settings update failed: %s", e, exc_info=True) + logger.error("Settings update failed: %s", e, exc_info=True) # noqa: G201 raise HTTPException( status_code=500, detail="Failed to update settings" ) from e diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_case_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_case_raw_repository.py index 3fefb49e..7929f043 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_case_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_case_raw_repository.py @@ -55,8 +55,8 @@ async def append_experience( vec = await vs.get_embedding(record.task_intent) record.vector = vec.tolist() if hasattr(vec, "tolist") else list(vec) record.vector_model = vs.get_model_name() - except Exception as e: - logger.error(f"[AgentCaseRepo] Auto-vectorize failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Auto-vectorize failed: {e}") # noqa: G004 if not record.vector: logger.warning( @@ -67,12 +67,12 @@ async def append_experience( try: result = await record.insert(session=session) logger.debug( - f"[AgentCaseRepo] Inserted experience: id={result.id}, " + f"[AgentCaseRepo] Inserted experience: id={result.id}, " # noqa: G004 f"intent='{(result.task_intent or '')[:80]}'" ) return result - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to insert experience: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to insert experience: {e}") # noqa: G004 return None async def get_by_event_id( @@ -82,8 +82,8 @@ async def get_by_event_id( try: object_id = ObjectId(event_id) return await self.model.find_one({"_id": object_id}, session=session) - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to get by event_id: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to get by event_id: {e}") # noqa: G004 return None async def get_by_ids( @@ -98,8 +98,8 @@ async def get_by_ids( return [] try: return await self.model.find(query_filter, session=session).to_list() - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to get by ids: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to get by ids: {e}") # noqa: G004 return [] async def get_by_parent_id( @@ -108,8 +108,8 @@ async def get_by_parent_id( """Retrieve agent case linked to a specific MemCell.""" try: return await self.model.find_one({"parent_id": parent_id}, session=session) - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to get by parent_id: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to get by parent_id: {e}") # noqa: G004 return None async def get_by_parent_ids( @@ -124,8 +124,8 @@ async def get_by_parent_ids( {"parent_id": {"$in": parent_ids}}, session=session ).to_list() return results - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to get by parent_ids: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to get by parent_ids: {e}") # noqa: G004 return [] async def find_by_filters( @@ -207,8 +207,8 @@ async def find_by_filters( len(results), ) return results - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to find by filters: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to find by filters: {e}") # noqa: G004 return [] async def count_by_filters( @@ -263,8 +263,8 @@ async def count_by_filters( count, ) return count - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to count by filters: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to count by filters: {e}") # noqa: G004 return 0 async def delete_by_user_id( @@ -289,8 +289,8 @@ async def delete_by_user_id( count, ) return count - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to delete by user_id: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to delete by user_id: {e}") # noqa: G004 return 0 async def delete_by_filters( @@ -344,8 +344,8 @@ async def delete_by_filters( count, ) return count - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to delete by filters: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to delete by filters: {e}") # noqa: G004 return 0 async def fetch_task_intents_by_event_ids( @@ -376,8 +376,8 @@ async def fetch_task_intents_by_event_ids( if case.parent_id and case.task_intent: result[case.parent_id] = case.task_intent return result - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed to fetch task intents: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed to fetch task intents: {e}") # noqa: G004 return {} async def find_by_filter_paginated( @@ -420,6 +420,6 @@ async def find_by_filter_paginated( len(results), ) return results - except Exception as e: - logger.error(f"[AgentCaseRepo] Failed paginated query: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseRepo] Failed paginated query: {e}") # noqa: G004 return [] diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py index c231db99..3c728751 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/agent_skill_raw_repository.py @@ -63,8 +63,8 @@ async def find_by_ids( # id_filter is {"_id": {"$in": [...]}}; merge directly query = {**id_filter, "confidence": {"$gte": min_confidence}} return await self.model.find(query, session=session).to_list() - except Exception as e: - logger.error(f"[AgentSkillRepo] Failed to find by ids: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentSkillRepo] Failed to find by ids: {e}") # noqa: G004 return [] async def save_skill( @@ -74,12 +74,12 @@ async def save_skill( try: result = await record.insert(session=session) logger.debug( - f"[AgentSkillRepo] Inserted skill: id={result.id}, " + f"[AgentSkillRepo] Inserted skill: id={result.id}, " # noqa: G004 f"cluster={result.cluster_id}, name='{result.name}'" ) return result - except Exception as e: - logger.error(f"[AgentSkillRepo] Failed to insert skill: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentSkillRepo] Failed to insert skill: {e}") # noqa: G004 return None async def get_by_cluster_id( @@ -106,8 +106,8 @@ async def get_by_cluster_id( query["confidence"] = {"$gte": min_confidence} results = await self.model.find(query, session=session).to_list() return results - except Exception as e: - logger.error(f"[AgentSkillRepo] Failed to get by cluster_id: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentSkillRepo] Failed to get by cluster_id: {e}") # noqa: G004 return [] async def update_skill_by_id( @@ -137,16 +137,16 @@ async def update_skill_by_id( ) if result.modified_count > 0: logger.debug( - f"[AgentSkillRepo] Updated skill id={record_id}, " + f"[AgentSkillRepo] Updated skill id={record_id}, " # noqa: G004 f"fields={list(updates.keys())}" ) return True logger.warning( - f"[AgentSkillRepo] No document matched for update id={record_id}" + f"[AgentSkillRepo] No document matched for update id={record_id}" # noqa: G004 ) return False - except Exception as e: - logger.error(f"[AgentSkillRepo] Failed to update skill id={record_id}: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentSkillRepo] Failed to update skill id={record_id}: {e}") # noqa: G004 return False async def soft_delete_by_id( @@ -171,15 +171,15 @@ async def soft_delete_by_id( session=session, ) if result.modified_count > 0: - logger.debug(f"[AgentSkillRepo] Soft-deleted skill id={record_id}") + logger.debug(f"[AgentSkillRepo] Soft-deleted skill id={record_id}") # noqa: G004 return True logger.warning( - f"[AgentSkillRepo] No document matched for soft-delete id={record_id}" + f"[AgentSkillRepo] No document matched for soft-delete id={record_id}" # noqa: G004 ) return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"[AgentSkillRepo] Failed to soft-delete skill id={record_id}: {e}" + f"[AgentSkillRepo] Failed to soft-delete skill id={record_id}: {e}" # noqa: G004 ) return False @@ -230,8 +230,8 @@ async def find_by_filters( .to_list() ) return results - except Exception as e: - logger.error(f"[AgentSkillRepo] Failed to find by filters: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentSkillRepo] Failed to find by filters: {e}") # noqa: G004 return [] async def count_by_filters( @@ -267,6 +267,6 @@ async def count_by_filters( count, ) return count - except Exception as e: - logger.error(f"[AgentSkillRepo] Failed to count by filters: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentSkillRepo] Failed to count by filters: {e}") # noqa: G004 return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/atomic_fact_record_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/atomic_fact_record_raw_repository.py index 47ec87a5..a7ef6fd5 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/atomic_fact_record_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/atomic_fact_record_raw_repository.py @@ -61,7 +61,7 @@ async def save( record.parent_id, ) return record - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to save atomic fact record: %s", e) return None @@ -105,7 +105,7 @@ async def get_by_id( else: logger.debug("ℹ️ Personal atomic fact not found: id=%s", log_id) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve personal atomic fact by ID: %s", e) return None @@ -138,7 +138,7 @@ async def find_by_ids( query_filter, projection_model=projection_model, session=session ).to_list() return await self.model.find(query_filter, session=session).to_list() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to find atomic facts by ids: %s", e) return [] @@ -187,7 +187,7 @@ async def get_by_parent_id( target_model.__name__, ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Failed to retrieve atomic facts by parent episodic memory ID: %s", e ) @@ -297,7 +297,7 @@ async def find_by_filters( target_model.__name__, ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve atomic facts: %s", e) return [] @@ -356,7 +356,7 @@ async def count_by_filters( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count atomic facts: %s", e) return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_data_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_data_raw_repository.py index 7215ac77..11bc5e22 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_data_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_data_raw_repository.py @@ -202,7 +202,7 @@ async def save_conversation_data( ) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Window accumulation confirmation failed: group_id=%s, session_id=%s, error=%s", group_id, @@ -281,7 +281,7 @@ async def get_conversation_data( ) return raw_data_list - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Conversation data fetch failed: group_id=%s, session_id=%s, error=%s", group_id, @@ -332,7 +332,7 @@ async def delete_conversation_data( ) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to mark conversation data as used: group_id=%s, session_id=%s, error=%s", group_id, @@ -395,7 +395,7 @@ async def fetch_unprocessed_conversation_data( ) return raw_data_list - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Unprocessed conversation data fetch failed: group_id=%s, session_id=%s, error=%s", group_id, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_status_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_status_raw_repository.py index d97f7278..e004c4bd 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_status_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/conversation_status_raw_repository.py @@ -43,7 +43,7 @@ async def get_by_group_id( else: logger.debug("⚠️ Conversation status not found: group_id=%s", group_id) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve conversation status by group ID: %s", e) return None @@ -64,7 +64,7 @@ async def delete_by_group_id( "✅ Successfully deleted conversation status by group ID: %s", group_id ) return True - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to delete conversation status by group ID: %s", e) return False @@ -160,7 +160,7 @@ async def upsert_by_group_id( # Other types of creation errors, re-raise raise create_error - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to update or create conversation status: %s", e) return None @@ -180,6 +180,6 @@ async def count_by_group_id( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count conversation statuses: %s", e) return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/episodic_memory_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/episodic_memory_raw_repository.py index 08dca34e..722573bb 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/episodic_memory_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/episodic_memory_raw_repository.py @@ -64,7 +64,7 @@ async def get_by_event_id( user_id, ) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Failed to retrieve episodic memory by event ID and user ID: %s", e ) @@ -96,8 +96,8 @@ async def get_by_event_ids( for event_id in event_ids: try: object_ids.append(ObjectId(event_id)) - except Exception as e: - logger.warning(f"⚠️ Invalid event_id: {event_id}, error: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"⚠️ Invalid event_id: {event_id}, error: {e}") # noqa: G004 continue if not object_ids: @@ -120,7 +120,7 @@ async def get_by_event_ids( len(result_dict), ) return result_dict - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to batch retrieve episodic memories: %s", e) return {} @@ -154,7 +154,7 @@ async def find_by_ids( query_filter, projection_model=projection_model, session=session ).to_list() return await self.model.find(query_filter, session=session).to_list() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to find episodic memories by ids: %s", e) return [] @@ -262,7 +262,7 @@ async def find_by_filters( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve episodic memories: %s", e) return [] @@ -321,7 +321,7 @@ async def count_by_filters( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count episodic memories: %s", e) return 0 @@ -350,7 +350,7 @@ async def append_episodic_memory( episodic_memory.vector = vector.tolist() # Set vectorization model information episodic_memory.vector_model = self.vectorize_service.get_model_name() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to synchronize vector: %s", e) try: await episodic_memory.insert(session=session) @@ -360,7 +360,7 @@ async def append_episodic_memory( episodic_memory.user_id, ) return episodic_memory - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to append episodic memory: %s", e) return None @@ -404,7 +404,7 @@ async def delete_by_event_id( user_id, ) return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Failed to delete episodic memory by event ID and user ID: %s", e ) @@ -432,7 +432,7 @@ async def delete_by_user_id( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to delete episodic memories by user ID: %s", e) return 0 @@ -550,7 +550,7 @@ async def find_by_filter_paginated( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to paginate query of EpisodicMemory: %s", e) return [] diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/foresight_record_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/foresight_record_raw_repository.py index 0a5f5e2d..cfbe62f8 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/foresight_record_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/foresight_record_raw_repository.py @@ -61,7 +61,7 @@ async def save( foresight.parent_id, ) return foresight - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to save personal foresight: %s", e) return None @@ -105,7 +105,7 @@ async def get_by_id( else: logger.debug("ℹ️ Personal foresight not found: id=%s", memory_id) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve personal foresight by ID: %s", e) return None @@ -154,7 +154,7 @@ async def get_by_parent_id( target_model.__name__, ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "❌ Failed to retrieve foresights by parent episodic memory ID: %s", e ) @@ -275,7 +275,7 @@ async def find_by_filters( target_model.__name__, ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve foresights: %s", e) return [] @@ -341,7 +341,7 @@ async def count_by_filters( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count foresights: %s", e) return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/group_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/group_raw_repository.py index f3c18609..2abc70c5 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/group_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/group_raw_repository.py @@ -34,7 +34,7 @@ async def get_by_group_id(self, group_id: str) -> Optional[Group]: else: logger.debug("Group not found: group_id=%s", group_id) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to get group by group_id: %s", e) return None @@ -95,6 +95,6 @@ async def upsert_by_group_id( else: raise create_error - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to upsert group: %s", e) return None diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py index 02166256..7f174dd3 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/mem_scene_raw_repository.py @@ -54,9 +54,9 @@ async def clear(self, group_id: Optional[str] = None) -> bool: async def get_by_group_id(self, group_id: str) -> Optional[MemScene]: try: return await self.model.find_one(MemScene.group_id == group_id) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"Failed to retrieve mem scene: group_id={group_id}, error={e}" + f"Failed to retrieve mem scene: group_id={group_id}, error={e}" # noqa: G004 ) return None @@ -71,16 +71,16 @@ async def upsert_by_group_id( if hasattr(existing, key): setattr(existing, key, value) await existing.save() - logger.debug(f"Updated mem scene: group_id={group_id}") + logger.debug(f"Updated mem scene: group_id={group_id}") # noqa: G004 return existing else: state["group_id"] = group_id mem_scene = MemScene(**state) await mem_scene.insert() - logger.info(f"Created mem scene: group_id={group_id}") + logger.info(f"Created mem scene: group_id={group_id}") # noqa: G004 return mem_scene - except Exception as e: - logger.error(f"Failed to save mem scene: group_id={group_id}, error={e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to save mem scene: group_id={group_id}, error={e}") # noqa: G004 return None async def get_cluster_assignments(self, group_id: str) -> Dict[str, str]: @@ -91,9 +91,9 @@ async def get_cluster_assignments(self, group_id: str) -> Dict[str, str]: # Derive eventid_to_cluster from memcell_info memcell_info = mem_scene.memcell_info or {} return {eid: info.get("memscene", "") for eid, info in memcell_info.items()} - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"Failed to retrieve cluster assignments: group_id={group_id}, error={e}" + f"Failed to retrieve cluster assignments: group_id={group_id}, error={e}" # noqa: G004 ) return {} @@ -102,18 +102,18 @@ async def delete_by_group_id(self, group_id: str) -> bool: mem_scene = await self.model.find_one(MemScene.group_id == group_id) if mem_scene: await mem_scene.delete() - logger.info(f"Deleted mem scene: group_id={group_id}") + logger.info(f"Deleted mem scene: group_id={group_id}") # noqa: G004 return True - except Exception as e: - logger.error(f"Failed to delete mem scene: group_id={group_id}, error={e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to delete mem scene: group_id={group_id}, error={e}") # noqa: G004 return False async def delete_all(self) -> int: try: result = await self.model.delete_all() count = result.deleted_count if result else 0 - logger.info(f"Deleted all mem scenes: {count} items") + logger.info(f"Deleted all mem scenes: {count} items") # noqa: G004 return count - except Exception as e: - logger.error(f"Failed to delete all mem scenes: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to delete all mem scenes: {e}") # noqa: G004 return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py index e2954bd5..6c89da38 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/memcell_raw_repository.py @@ -57,7 +57,7 @@ async def get_by_event_id(self, event_id: str) -> Optional[MemCell]: else: logger.debug("⚠️ MemCell not found: event_id=%s", event_id) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve MemCell by event_id: %s", e) return None @@ -89,7 +89,7 @@ async def get_by_event_ids( try: object_ids.append(ObjectId(event_id)) valid_event_ids.append(event_id) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("⚠️ Invalid event_id: %s, error: %s", event_id, e) if not object_ids: @@ -119,7 +119,7 @@ async def get_by_event_ids( return result_dict - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to batch retrieve MemCell by event_ids: %s", e) return {} @@ -133,7 +133,7 @@ async def append_memcell( await memcell.insert(session=session) logger.debug("Successfully appended MemCell: %s", memcell.event_id) return memcell - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to append MemCell: %s", e) return None @@ -196,7 +196,7 @@ async def delete_by_event_id( ) return True return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to soft delete MemCell by event_id: %s", e) return False @@ -224,7 +224,7 @@ async def hard_delete_by_event_id( ) return True return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to hard delete MemCell by event_id: %s", e) return False @@ -271,7 +271,7 @@ async def find_by_user_id( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to query MemCell by user ID: %s", e) return [] @@ -329,7 +329,7 @@ async def find_by_user_and_time_range( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to query MemCell by user and time range: %s", e) return [] @@ -372,7 +372,7 @@ async def find_by_group_id( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to query MemCell by group ID: %s", e) return [] @@ -422,7 +422,7 @@ async def find_by_time_range( return results except Exception as e: logger.error("❌ Failed to query MemCell by time range: %s", e) - logger.error("Detailed error information:", exc_info=True) + logger.error("Detailed error information:", exc_info=True) # noqa: G201 return [] async def find_by_participants( @@ -467,7 +467,7 @@ async def find_by_participants( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to query MemCell by participants: %s", e) return [] @@ -511,7 +511,7 @@ async def search_by_keywords( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to query MemCell by keywords: %s", e) return [] @@ -545,7 +545,7 @@ async def delete_by_user_id( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to soft delete all MemCell of user: %s", e) return 0 @@ -577,7 +577,7 @@ async def delete_by_group_id( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to soft delete all MemCell of group: %s", e) return 0 @@ -665,7 +665,7 @@ async def hard_delete_by_user_id( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to hard delete all MemCell of user: %s", e) return 0 @@ -707,7 +707,7 @@ async def delete_by_time_range( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to soft delete MemCell within time range: %s", e) return 0 @@ -747,7 +747,7 @@ async def hard_delete_by_time_range( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to hard delete MemCell within time range: %s", e) return 0 @@ -776,7 +776,7 @@ async def restore_by_event_id( ) return True return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to restore MemCell by event_id: %s", e) return False @@ -804,7 +804,7 @@ async def restore_by_user_id( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to restore all MemCell of user: %s", e) return 0 @@ -845,7 +845,7 @@ async def restore_by_time_range( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to restore MemCell within time range: %s", e) return 0 @@ -869,7 +869,7 @@ async def count_by_user_id(self, user_id: str) -> int: count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count user MemCell: %s", e) return 0 @@ -905,7 +905,7 @@ async def count_by_time_range( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count MemCell within time range: %s", e) return 0 @@ -933,7 +933,7 @@ async def get_latest_by_user(self, user_id: str, limit: int = 10) -> List[MemCel len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve latest user MemCell: %s", e) return [] diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/raw_message_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/raw_message_repository.py index 8b888bae..6397795f 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/raw_message_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/raw_message_repository.py @@ -54,7 +54,7 @@ async def save( raw_message.request_id, ) return raw_message - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to save raw message: %s", e) return None @@ -150,7 +150,7 @@ def _parse_create_time(create_time: Any) -> Optional[str]: from common_utils.datetime_utils import to_iso_format return to_iso_format(create_time) - except Exception: + except Exception: # noqa: BLE001 if isinstance(create_time, str): return create_time return None @@ -175,7 +175,7 @@ async def get_by_request_id( {"request_id": request_id}, session=session ) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to get raw message by request ID: %s", e) return None @@ -218,7 +218,7 @@ async def find_one_by_group_sender_message( message_id, ) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to find raw message by group_id/sender_id/message_id: " "group_id=%s, sender_id=%s, message_id=%s, error=%s", @@ -285,7 +285,7 @@ async def find_by_group_id( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to query raw messages by group_id: %s", e) return [] @@ -356,7 +356,7 @@ async def find_by_group_id_with_statuses( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to query raw messages by group_id with statuses: %s", e ) @@ -392,7 +392,7 @@ async def find_by_sender_id( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to query raw messages by sender_id: %s", e) return [] @@ -418,7 +418,7 @@ async def delete_by_group_id( "Deleted raw messages: group_id=%s, deleted=%d", group_id, deleted_count ) return deleted_count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to delete raw messages: group_id=%s, error=%s", group_id, e ) @@ -468,7 +468,7 @@ async def delete_by_filters( "Soft deleted raw messages: filter=%s, deleted=%d", filter_dict, count ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to soft delete raw messages: filter=%s, error=%s", {"sender_id": sender_id, "group_id": group_id}, @@ -510,7 +510,7 @@ async def confirm_accumulation_by_group_id( modified_count, ) return modified_count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to confirm window accumulation: group_id=%s, error=%s", group_id, @@ -564,7 +564,7 @@ async def confirm_accumulation_by_message_ids( modified_count, ) return modified_count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to confirm window accumulation (precise): group_id=%s, error=%s", group_id, @@ -614,7 +614,7 @@ async def mark_as_used_by_group_id( modified_count, ) return modified_count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to mark as used: group_id=%s, error=%s", group_id, e) return 0 @@ -707,7 +707,7 @@ async def find_pending_by_filters( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to query pending raw messages: sender_id=%s, group_ids=%s, error=%s", sender_id, diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/sender_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/sender_raw_repository.py index bd794844..cc5431bd 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/sender_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/sender_raw_repository.py @@ -34,7 +34,7 @@ async def get_by_sender_id(self, sender_id: str) -> Optional[Sender]: else: logger.debug("Sender not found: sender_id=%s", sender_id) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to get sender by sender_id: %s", e) return None @@ -57,7 +57,7 @@ async def get_by_sender_ids(self, sender_ids: List[str]) -> List[Sender]: "Batch retrieved %d senders for %d ids", len(results), len(sender_ids) ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to batch get senders: %s", e) return [] @@ -120,6 +120,6 @@ async def upsert_by_sender_id( else: raise create_error - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to upsert sender: %s", e) return None diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/session_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/session_raw_repository.py index 7eded23f..8649a532 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/session_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/session_raw_repository.py @@ -34,7 +34,7 @@ async def get_by_session_id(self, session_id: str) -> Optional[Session]: else: logger.debug("Session not found: session_id=%s", session_id) return result - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to get session by session_id: %s", e) return None @@ -98,6 +98,6 @@ async def upsert_by_session_id( else: raise create_error - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to upsert session: %s", e) return None diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/settings_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/settings_raw_repository.py index f7993dab..c74ea53c 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/settings_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/settings_raw_repository.py @@ -52,7 +52,7 @@ async def get_global_settings( if doc: logger.debug("Retrieved global settings") return doc - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("Failed to retrieve global settings: %s", e) return None @@ -90,7 +90,7 @@ async def upsert_global_settings( logger.info("Created new global settings") return new_doc except Exception as create_error: - logger.error( + logger.error( # noqa: G201 "Failed to create global settings: %s", create_error, exc_info=True ) return None @@ -98,7 +98,7 @@ async def upsert_global_settings( except ValidationException: raise except Exception as e: - logger.error("Failed to upsert global settings: %s", e, exc_info=True) + logger.error("Failed to upsert global settings: %s", e, exc_info=True) # noqa: G201 return None async def update_global_settings( @@ -129,5 +129,5 @@ async def update_global_settings( except ValidationException: raise except Exception as e: - logger.error("Failed to update global settings: %s", e, exc_info=True) + logger.error("Failed to update global settings: %s", e, exc_info=True) # noqa: G201 return None diff --git a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py index 58718a53..6a8aee22 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/persistence/repository/user_profile_raw_repository.py @@ -100,18 +100,18 @@ async def get_by_user_and_group( return await self.model.find_one( UserProfile.user_id == user_id, UserProfile.group_id == group_id ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"Failed to retrieve user profile: user_id={user_id}, group_id={group_id}, error={e}" + f"Failed to retrieve user profile: user_id={user_id}, group_id={group_id}, error={e}" # noqa: G004 ) return None async def get_all_by_group(self, group_id: str) -> List[UserProfile]: try: return await self.model.find(UserProfile.group_id == group_id).to_list() - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"Failed to retrieve group user profiles: group_id={group_id}, error={e}" + f"Failed to retrieve group user profiles: group_id={group_id}, error={e}" # noqa: G004 ) return [] @@ -123,8 +123,8 @@ async def get_all_by_user(self, user_id: str, limit: int = 40) -> List[UserProfi .limit(limit) .to_list() ) - except Exception as e: - logger.error(f"Failed to get user profile: user_id={user_id}, error={e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to get user profile: user_id={user_id}, error={e}") # noqa: G004 return [] async def find_by_filters( @@ -211,7 +211,7 @@ async def find_by_filters( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to retrieve user profiles: %s", e) return [] @@ -262,7 +262,7 @@ async def count_by_filters( count, ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("❌ Failed to count user profiles: %s", e) return 0 @@ -304,7 +304,7 @@ async def upsert( await existing.save() logger.debug( - f"Updated user profile: user_id={user_id}, group_id={group_id}, update_count={existing.update_count}" + f"Updated user profile: user_id={user_id}, group_id={group_id}, update_count={existing.update_count}" # noqa: G004 ) saved_profile = existing else: @@ -320,7 +320,7 @@ async def upsert( ) await user_profile.insert() logger.info( - f"Created user profile: user_id={user_id}, group_id={group_id}" + f"Created user profile: user_id={user_id}, group_id={group_id}" # noqa: G004 ) saved_profile = user_profile @@ -332,9 +332,9 @@ async def upsert( return saved_profile - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"Failed to save user profile: user_id={user_id}, group_id={group_id}, error={e}" + f"Failed to save user profile: user_id={user_id}, group_id={group_id}, error={e}" # noqa: G004 ) return None @@ -366,14 +366,14 @@ async def _trigger_milvus_indexing( stats = await index_user_profile(user_id, group_id, profile, doc_id=doc_id) logger.info( - f"✅ Profile Milvus indexing completed: user_id={user_id}, group_id={group_id}, " + f"✅ Profile Milvus indexing completed: user_id={user_id}, group_id={group_id}, " # noqa: G004 f"deleted={stats.get('deleted_count', 0)}, indexed={stats.get('total_count', 0)}" ) except Exception as e: # Log error but don't fail the main operation - logger.error( - f"❌ Failed to trigger Milvus indexing: user_id={user_id}, group_id={group_id}, error={e}", + logger.error( # noqa: G201 + f"❌ Failed to trigger Milvus indexing: user_id={user_id}, group_id={group_id}, error={e}", # noqa: G004 exc_info=True, ) @@ -382,12 +382,12 @@ async def delete_by_group(self, group_id: str) -> int: result = await self.model.find(UserProfile.group_id == group_id).delete() count = result.deleted_count if result else 0 logger.info( - f"Deleted group user profiles: group_id={group_id}, count={count}" + f"Deleted group user profiles: group_id={group_id}, count={count}" # noqa: G004 ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( - f"Failed to delete group user profiles: group_id={group_id}, error={e}" + f"Failed to delete group user profiles: group_id={group_id}, error={e}" # noqa: G004 ) return 0 @@ -395,8 +395,8 @@ async def delete_all(self) -> int: try: result = await self.model.delete_all() count = result.deleted_count if result else 0 - logger.info(f"Deleted all user profiles: {count} items") + logger.info(f"Deleted all user profiles: {count} items") # noqa: G004 return count - except Exception as e: - logger.error(f"Failed to delete all user profiles: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Failed to delete all user profiles: {e}") # noqa: G004 return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/foresight_milvus_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/foresight_milvus_converter.py index 4f10339e..631f0851 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/foresight_milvus_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/foresight_milvus_converter.py @@ -44,9 +44,9 @@ def _parse_time_field(cls, time_value, field_name: str, doc_id) -> int: return int(dt.timestamp() * 1000) elif isinstance(time_value, (int, float)): return int(time_value * 1000) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( - f"Failed to parse {field_name} (doc_id={doc_id}): {time_value}, error: {e}" + f"Failed to parse {field_name} (doc_id={doc_id}): {time_value}, error: {e}" # noqa: G004 ) return 0 diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py index c3c6018e..5f1d2d85 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/milvus/converter/user_profile_milvus_converter.py @@ -144,7 +144,7 @@ def _make_entity(embed_text: str, item_type: str) -> Dict[str, Any]: return entities except Exception as e: - logger.error( + logger.error( # noqa: G201 "Failed to convert MongoDB UserProfile to Milvus entities: %s", e, exc_info=True, diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/repository/agent_skill_milvus_repository.py b/methods/EverCore/src/infra_layer/adapters/out/search/repository/agent_skill_milvus_repository.py index c0718ef6..aa5d1ae7 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/repository/agent_skill_milvus_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/repository/agent_skill_milvus_repository.py @@ -151,7 +151,7 @@ async def delete_by_cluster_id(self, cluster_id: str) -> int: "Deleted %d Milvus records for cluster=%s", count, cluster_id ) return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to delete Milvus records for cluster=%s: %s", cluster_id, e ) diff --git a/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py b/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py index 2b2b84c4..17feb6e3 100644 --- a/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py +++ b/methods/EverCore/src/infra_layer/adapters/out/search/repository/user_profile_milvus_repository.py @@ -147,7 +147,7 @@ async def delete_by_user_group(self, user_id: str, group_id: str) -> int: return count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to delete profile items: user_id=%s, group_id=%s, error=%s", user_id, diff --git a/methods/EverCore/src/memory_layer/cluster_manager/manager.py b/methods/EverCore/src/memory_layer/cluster_manager/manager.py index 81da6f70..e29f568f 100644 --- a/methods/EverCore/src/memory_layer/cluster_manager/manager.py +++ b/methods/EverCore/src/memory_layer/cluster_manager/manager.py @@ -233,8 +233,8 @@ def __init__( if VECTORIZE_SERVICE_AVAILABLE: try: self._vectorize_service = get_vectorize_service() - except Exception as e: - logger.warning(f"Failed to initialize vectorize service: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to initialize vectorize service: {e}") # noqa: G004 # LLM provider (for llm algorithm) self._llm_provider = llm_provider @@ -302,7 +302,7 @@ async def _cluster_memcell_embedding( vector = await self._get_embedding(text) if vector is None or vector.size == 0: logger.warning( - f"Failed to get embedding for event {event_id}, creating singleton cluster" + f"Failed to get embedding for event {event_id}, creating singleton cluster" # noqa: G004 ) cluster_id = state.assign_new_cluster(event_id) state.event_ids.append(event_id) @@ -442,7 +442,7 @@ async def _cluster_memcell_llm( self._append_event(state, event_id, vector, timestamp) self._stats["clustered_memcells"] += 1 logger.info( - f"[LLM Clustering] First case cluster: {event_id} -> {cluster_id}" + f"[LLM Clustering] First case cluster: {event_id} -> {cluster_id}" # noqa: G004 ) return cluster_id, state @@ -457,7 +457,7 @@ async def _cluster_memcell_llm( candidate_ids = [cid for cid, _ in scored_candidates] top1_sim = scored_candidates[0][1] if scored_candidates else -1.0 logger.info( - f"[LLM Clustering] Embedding recall: {len(candidate_ids)} candidates " + f"[LLM Clustering] Embedding recall: {len(candidate_ids)} candidates " # noqa: G004 f"(top1_sim={top1_sim:.3f}), " f"from {len(state.case_cluster_ids)} case clusters" ) @@ -469,7 +469,7 @@ async def _cluster_memcell_llm( self._append_event(state, event_id, vector, timestamp) self._stats["clustered_memcells"] += 1 logger.info( - f"[LLM Clustering] Fast path: {event_id} -> {cluster_id} " + f"[LLM Clustering] Fast path: {event_id} -> {cluster_id} " # noqa: G004 f"(sim={top1_sim:.3f} >= {self.config.llm_skip_threshold})" ) return cluster_id, state @@ -490,7 +490,7 @@ async def _cluster_memcell_llm( if llm_result is None: logger.warning( - f"[LLM Clustering] LLM call failed for event {event_id}, " + f"[LLM Clustering] LLM call failed for event {event_id}, " # noqa: G004 f"falling back to embedding top-1" ) # Fall back to embedding: use top-1 candidate if available, else new cluster @@ -521,7 +521,7 @@ async def _cluster_memcell_llm( self._stats["clustered_memcells"] += 1 reason = llm_result.get("reason", "") if llm_result else "" logger.info( - f"[LLM Clustering] 🎯 Event {event_id} -> {cluster_id} " + f"[LLM Clustering] 🎯 Event {event_id} -> {cluster_id} " # noqa: G004 f"| intent: {text} | reason: {reason}" ) return cluster_id, state @@ -642,10 +642,10 @@ async def _call_llm_for_clustering(self, prompt: str) -> Optional[Dict[str, Any] if data and "cluster_id" in data: return data logger.warning( - f"[LLM Clustering] Retry {attempt + 1}/3: invalid response format" + f"[LLM Clustering] Retry {attempt + 1}/3: invalid response format" # noqa: G004 ) - except Exception as e: - logger.warning(f"[LLM Clustering] Retry {attempt + 1}/3: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"[LLM Clustering] Retry {attempt + 1}/3: {e}") # noqa: G004 return None def _find_best_cluster( @@ -701,8 +701,8 @@ async def _get_embedding(self, text: str) -> Optional[np.ndarray]: vector_arr = await self._vectorize_service.get_embedding(text) if vector_arr is not None: return np.array(vector_arr, dtype=np.float32) - except Exception as e: - logger.warning(f"Failed to get embedding: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to get embedding: {e}") # noqa: G004 return None @@ -748,8 +748,8 @@ def _parse_timestamp(self, timestamp: Any) -> Optional[float]: dt = from_iso_format(timestamp) return dt.timestamp() - except Exception as e: - logger.warning(f"Failed to parse timestamp {timestamp}: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Failed to parse timestamp {timestamp}: {e}") # noqa: G004 return None @@ -763,8 +763,8 @@ async def _notify_callbacks( await callback(group_id, memcell, cluster_id) else: callback(group_id, memcell, cluster_id) - except Exception as e: - logger.error(f"Callback error: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Callback error: {e}") # noqa: G004 def get_stats(self) -> Dict[str, Any]: """Get clustering statistics.""" diff --git a/methods/EverCore/src/memory_layer/llm/openai_provider.py b/methods/EverCore/src/memory_layer/llm/openai_provider.py index 96ac717f..522e1b76 100644 --- a/methods/EverCore/src/memory_layer/llm/openai_provider.py +++ b/methods/EverCore/src/memory_layer/llm/openai_provider.py @@ -148,7 +148,7 @@ def _report_token_usage(self, prompt_tokens: int, completion_tokens: int) -> Non try: collector = get_bean_by_type(TokenUsageCollector) collector.add(self.model, prompt_tokens, completion_tokens, call_type="llm") - except Exception: + except Exception: # noqa: BLE001 pass def _log_completion_metrics(self, response_data: dict, duration: float) -> None: @@ -318,7 +318,7 @@ async def test_connection(self) -> bool: self.model, ) return success - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "\u274c [OpenAI-%s] API connection test failed: %s", self.model, e ) diff --git a/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py b/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py index 424dc34e..7652807d 100644 --- a/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py +++ b/methods/EverCore/src/memory_layer/memcell_extractor/conv_memcell_extractor.py @@ -265,7 +265,7 @@ def _create_memcell_directly( ) logger.info( - f"[ConvMemCellExtractor] ✅ MemCell created: " + f"[ConvMemCellExtractor] ✅ MemCell created: " # noqa: G004 f"messages={len(messages)}, trigger={trigger_type}" ) @@ -308,7 +308,7 @@ def _format_messages_with_indices(self, messages: List[Dict[str, Any]]) -> str: lines.append(f"[{i}] {sender_name}: {content}") else: logger.debug( - f"[ConvMemCellExtractor] Warning: message {i} has no content" + f"[ConvMemCellExtractor] Warning: message {i} has no content" # noqa: G004 ) return "\n".join(lines) @@ -365,7 +365,7 @@ def _parse_batch_boundary_response( boundaries.append(int(item)) except (TypeError, ValueError): logger.warning( - f"[ConvMemCellExtractor] Skipping unparseable boundary value: {item!r}" + f"[ConvMemCellExtractor] Skipping unparseable boundary value: {item!r}" # noqa: G004 ) return BatchBoundaryResult( @@ -390,7 +390,7 @@ async def _detect_boundaries( messages_text = self._format_messages_with_indices(messages) logger.debug( - f"[ConvMemCellExtractor] Detect boundaries – total messages: {len(messages)}, " + f"[ConvMemCellExtractor] Detect boundaries – total messages: {len(messages)}, " # noqa: G004 f"formatted text length: {len(messages_text)}" ) @@ -399,7 +399,7 @@ async def _detect_boundaries( ) logger.debug( - f"[ConvMemCellExtractor] === BOUNDARY DETECTION PROMPT ===\n{prompt}\n" + f"[ConvMemCellExtractor] === BOUNDARY DETECTION PROMPT ===\n{prompt}\n" # noqa: G004 f"[ConvMemCellExtractor] === END PROMPT ===" ) @@ -410,7 +410,7 @@ async def _detect_boundaries( for i in range(5): resp = await self.llm_provider.generate(prompt) logger.debug( - f"[ConvMemCellExtractor] === BOUNDARY DETECTION RESPONSE (attempt {i + 1}) ===\n" + f"[ConvMemCellExtractor] === BOUNDARY DETECTION RESPONSE (attempt {i + 1}) ===\n" # noqa: G004 f"{resp}\n" f"[ConvMemCellExtractor] === END RESPONSE ===" ) @@ -423,7 +423,7 @@ async def _detect_boundaries( ] if len(valid_boundaries) != len(result.boundaries): logger.warning( - f"[ConvMemCellExtractor] Filtered {len(result.boundaries) - len(valid_boundaries)} " + f"[ConvMemCellExtractor] Filtered {len(result.boundaries) - len(valid_boundaries)} " # noqa: G004 f"out-of-range boundaries (total messages: {len(messages)})" ) result.boundaries = sorted(valid_boundaries) @@ -441,7 +441,7 @@ async def _detect_boundaries( return result logger.warning( - f"[ConvMemCellExtractor] Failed to parse JSON from LLM response " + f"[ConvMemCellExtractor] Failed to parse JSON from LLM response " # noqa: G004 f"(attempt {i + 1}/5), response: {resp[:200]}..." ) @@ -484,7 +484,7 @@ async def extract_memcell( if not new_message_dict_list: if request.flush and history_message_dict_list: logger.info( - f"[ConvMemCellExtractor] Flush with no new messages: " + f"[ConvMemCellExtractor] Flush with no new messages: " # noqa: G004 f"packing {len(history_message_dict_list)} history messages into final MemCell" ) memcell = self._create_memcell_directly( @@ -518,7 +518,7 @@ async def extract_memcell( trigger_type = 'token_limit' if exceeds_token else 'message_limit' logger.debug( - f"[ConvMemCellExtractor] Force split triggered: " + f"[ConvMemCellExtractor] Force split triggered: " # noqa: G004 f"tokens={total_tokens}/{self.hard_token_limit}, " f"messages={total_messages}/{self.hard_message_limit}, " f"split_at={split_at}" @@ -554,7 +554,7 @@ async def extract_memcell( # === Phase 3: Flush tail === if request.flush and all_msgs: logger.info( - f"[ConvMemCellExtractor] Flush mode: packing {len(all_msgs)} remaining " + f"[ConvMemCellExtractor] Flush mode: packing {len(all_msgs)} remaining " # noqa: G004 f"messages into final MemCell" ) memcell = self._create_memcell_directly(all_msgs, request, 'flush') @@ -565,12 +565,12 @@ async def extract_memcell( if result_memcells: logger.info( - f"[ConvMemCellExtractor] ✅ Extracted {len(result_memcells)} MemCell(s), " + f"[ConvMemCellExtractor] ✅ Extracted {len(result_memcells)} MemCell(s), " # noqa: G004 f"remaining_msgs={len(all_msgs)}, should_wait={should_wait}" ) else: logger.debug( - f"[ConvMemCellExtractor] ⏳ No boundary detected, " + f"[ConvMemCellExtractor] ⏳ No boundary detected, " # noqa: G004 f"remaining_msgs={len(all_msgs)}, should_wait={should_wait}" ) diff --git a/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py index aaf49636..aff63ca7 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/agent_case_extractor.py @@ -212,7 +212,7 @@ def _heuristic_trim_tool_outputs( trimmed_count += 1 if trimmed_count > 0: logger.info( - f"[AgentCaseExtractor] Heuristic trim: " + f"[AgentCaseExtractor] Heuristic trim: " # noqa: G004 f"truncated {trimmed_count} content fields" ) return result @@ -267,7 +267,7 @@ async def _pre_compress_to_list( total_size = sum(s for _, _, s in groups_with_size) if total_size <= self.pre_compress_chunk_size: logger.debug( - f"[AgentCaseExtractor] Tool content {total_size} tokens " + f"[AgentCaseExtractor] Tool content {total_size} tokens " # noqa: G004 f"<= {self.pre_compress_chunk_size}, no compression needed" ) return items @@ -289,7 +289,7 @@ async def _pre_compress_to_list( ] logger.debug( - f"[AgentCaseExtractor] Selective compression: " + f"[AgentCaseExtractor] Selective compression: " # noqa: G004 f"{len(groups_to_compress)}/{len(tool_call_groups)} groups, " f"{total_size} total tokens" ) @@ -330,7 +330,7 @@ async def _pre_compress_to_list( for round_idx, result in enumerate(results): if isinstance(result, Exception): logger.warning( - f"[AgentCaseExtractor] Chunk {round_idx + 1} compression error: " + f"[AgentCaseExtractor] Chunk {round_idx + 1} compression error: " # noqa: G004 f"{result}, keeping original messages" ) all_compressed.extend(chunk_msg_lists[round_idx]) @@ -338,7 +338,7 @@ async def _pre_compress_to_list( all_compressed.extend(result) else: logger.warning( - f"[AgentCaseExtractor] Chunk {round_idx + 1} compression failed, " + f"[AgentCaseExtractor] Chunk {round_idx + 1} compression failed, " # noqa: G004 "keeping original messages" ) all_compressed.extend(chunk_msg_lists[round_idx]) @@ -351,7 +351,7 @@ async def _pre_compress_to_list( items[idx] = all_compressed[i] else: logger.warning( - f"[AgentCaseExtractor] Compressed count {len(all_compressed)} " + f"[AgentCaseExtractor] Compressed count {len(all_compressed)} " # noqa: G004 f"!= selected message count {len(selected_indices)}, keeping originals" ) @@ -380,12 +380,12 @@ async def _compress_tool_chunk( ): return data["compressed_messages"] logger.warning( - f"[AgentCaseExtractor] Tool pre-compress attempt {attempt + 1}/2: " + f"[AgentCaseExtractor] Tool pre-compress attempt {attempt + 1}/2: " # noqa: G004 f"invalid response format" ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( - f"[AgentCaseExtractor] Tool pre-compress attempt {attempt + 1}/2: {e}" + f"[AgentCaseExtractor] Tool pre-compress attempt {attempt + 1}/2: {e}" # noqa: G004 ) return None @@ -400,10 +400,10 @@ async def _filter_conversation(self, messages_json: str) -> bool: worth = data["worth_extracting"] if not worth: reason = data.get("reason", "") - logger.info(f"[AgentCaseExtractor] Filtered out by LLM: {reason}") + logger.info(f"[AgentCaseExtractor] Filtered out by LLM: {reason}") # noqa: G004 return bool(worth) - except Exception as e: - logger.warning(f"[AgentCaseExtractor] Filter failed: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"[AgentCaseExtractor] Filter failed: {e}") # noqa: G004 # Default to extracting if filter fails return True @@ -430,12 +430,12 @@ async def _compress_experience( return None return data logger.warning( - f"[AgentCaseExtractor] Compress attempt {attempt + 1}/2: " + f"[AgentCaseExtractor] Compress attempt {attempt + 1}/2: " # noqa: G004 f"missing or invalid 'task_intent' field" ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( - f"[AgentCaseExtractor] Compress attempt {attempt + 1}/2: {e}" + f"[AgentCaseExtractor] Compress attempt {attempt + 1}/2: {e}" # noqa: G004 ) logger.error( @@ -566,8 +566,8 @@ async def _compute_embedding(self, text: str) -> Optional[Dict[str, Any]]: "embedding": vec.tolist() if hasattr(vec, "tolist") else list(vec), "vector_model": vs.get_model_name(), } - except Exception as e: - logger.error(f"[AgentCaseExtractor] Embedding failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseExtractor] Embedding failed: {e}") # noqa: G004 return None async def extract_memory( @@ -587,7 +587,7 @@ async def extract_memory( if memcell.type != RawDataType.AGENTCONVERSATION: logger.warning( - f"[AgentCaseExtractor] Expected AGENT_CONVERSATION, got {memcell.type}" + f"[AgentCaseExtractor] Expected AGENT_CONVERSATION, got {memcell.type}" # noqa: G004 ) return None @@ -600,7 +600,7 @@ async def extract_memory( # Pre-filter: skip conversations not worth extracting skip_reason = self._should_skip(original_data) if skip_reason: - logger.info(f"[AgentCaseExtractor] {skip_reason}, skipping") + logger.info(f"[AgentCaseExtractor] {skip_reason}, skipping") # noqa: G004 return None # Heuristic trim: truncate oversized tool outputs and assistant responses. @@ -613,7 +613,7 @@ async def extract_memory( ) ) logger.info( - f"[AgentCaseExtractor] event_id={memcell.event_id}, " + f"[AgentCaseExtractor] event_id={memcell.event_id}, " # noqa: G004 f"total_tokens={total_tokens}, message_count={len(original_data)}" ) @@ -635,7 +635,7 @@ async def extract_memory( 500, int(self.max_assistant_response_tokens * scale) ) logger.info( - f"[AgentCaseExtractor] Total tokens {total_tokens} > " + f"[AgentCaseExtractor] Total tokens {total_tokens} > " # noqa: G004 f"scale_trigger ({scale_trigger}), " f"scale={scale:.2f} -> trim limits: " f"tool_output={trim_tool_output}, tool_args={trim_tool_args}, " @@ -659,7 +659,7 @@ async def extract_memory( ) if trimmed_tokens > self.pre_compress_chunk_size * 2: logger.info( - f"[AgentCaseExtractor] Still {trimmed_tokens} tokens after trim " + f"[AgentCaseExtractor] Still {trimmed_tokens} tokens after trim " # noqa: G004 f"(> 2x PRE_COMPRESS_CHUNK_SIZE {self.pre_compress_chunk_size * 2}), skipping extraction" ) return None @@ -675,7 +675,7 @@ async def extract_memory( ) logger.debug( - f"[AgentCaseExtractor] Pre-compressed: " + f"[AgentCaseExtractor] Pre-compressed: " # noqa: G004 f"{len(pre_compressed_list)} items, {len(messages_json)} chars" ) @@ -705,7 +705,7 @@ async def extract_memory( ) if raw_intent != original_intent: logger.info( - f"[AgentCaseExtractor] Truncated task_intent to " + f"[AgentCaseExtractor] Truncated task_intent to " # noqa: G004 f"{MAX_TASK_INTENT_TOKENS} tokens, " f"original: {original_intent}" ) @@ -734,12 +734,12 @@ async def extract_memory( experience.vector_model = embedding_data["vector_model"] logger.debug( - f"[AgentCaseExtractor] Extracted: " + f"[AgentCaseExtractor] Extracted: " # noqa: G004 f"intent='{experience.task_intent[:80]}'" ) return experience - except Exception as e: - logger.error(f"[AgentCaseExtractor] Extraction failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentCaseExtractor] Extraction failed: {e}") # noqa: G004 return None diff --git a/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py index 3895fc37..b39eabe4 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/agent_skill_extractor.py @@ -290,8 +290,8 @@ async def _compute_embedding(self, text: str) -> Optional[Dict[str, Any]]: "embedding": vec.tolist() if hasattr(vec, "tolist") else list(vec), "vector_model": vs.get_model_name(), } - except Exception as e: - logger.error(f"[AgentSkillExtractor] Embedding failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"[AgentSkillExtractor] Embedding failed: {e}") # noqa: G004 return None def _select_prompt(self, case_records: List[AgentCase]) -> str: @@ -329,10 +329,10 @@ async def _call_llm( if data and isinstance(data.get("operations"), list): return data logger.warning( - f"[AgentSkillExtractor] LLM retry {attempt + 1}/3: invalid format" + f"[AgentSkillExtractor] LLM retry {attempt + 1}/3: invalid format" # noqa: G004 ) - except Exception as e: - logger.warning(f"[AgentSkillExtractor] LLM retry {attempt + 1}/3: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"[AgentSkillExtractor] LLM retry {attempt + 1}/3: {e}") # noqa: G004 return None async def _evaluate_maturity( @@ -377,7 +377,7 @@ async def _evaluate_maturity( data.get("reason", ""), ) return score - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("[AgentSkillExtractor] Maturity evaluation failed: %s", e) return None @@ -497,7 +497,7 @@ async def _apply_add( saved = await skill_repo.save_skill(record) if saved: logger.info( - f"[AgentSkillExtractor] ADD skill: name='{name}', cluster={cluster_id}" + f"[AgentSkillExtractor] ADD skill: name='{name}', cluster={cluster_id}" # noqa: G004 ) return saved @@ -537,14 +537,14 @@ async def _apply_update( index = int(op.get("index", -1)) except (ValueError, TypeError): logger.warning( - f"[AgentSkillExtractor] update index is not a valid integer: {op.get('index')!r}, skipping" + f"[AgentSkillExtractor] update index is not a valid integer: {op.get('index')!r}, skipping" # noqa: G004 ) return False data = op.get("data", {}) if index < 0 or index >= len(existing_skill_records): logger.warning( - f"[AgentSkillExtractor] update index {index} out of range " + f"[AgentSkillExtractor] update index {index} out of range " # noqa: G004 f"(valid: 0..{len(existing_skill_records) - 1} for {len(existing_skill_records)} skills), skipping" ) return False @@ -594,7 +594,7 @@ async def _apply_update( if not updates: logger.warning( - f"[AgentSkillExtractor] update operation for index {index} has no fields to update, skipping" + f"[AgentSkillExtractor] update operation for index {index} has no fields to update, skipping" # noqa: G004 ) return False @@ -739,7 +739,7 @@ async def _apply_update( record.updated_at = get_now_with_timezone() result.updated_records.append(record) logger.info( - f"[AgentSkillExtractor] UPDATE skill[{index}]: id={record_id}, " + f"[AgentSkillExtractor] UPDATE skill[{index}]: id={record_id}, " # noqa: G004 f"fields={list(updates.keys())}" ) return success @@ -783,7 +783,7 @@ async def _load_case_history( max_cases, ) return records[:max_cases] - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning("[AgentSkillExtractor] Failed to load case_history: %s", e) return [] @@ -818,14 +818,14 @@ async def extract_and_save( if not new_case_records: logger.debug( - f"[AgentSkillExtractor] No new cases for cluster={cluster_id}, skipping" + f"[AgentSkillExtractor] No new cases for cluster={cluster_id}, skipping" # noqa: G004 ) return empty_result # When too many existing skills, select top-k most relevant ones if len(existing_skill_records) > max_skills_in_prompt: logger.info( - f"[AgentSkillExtractor] {len(existing_skill_records)} existing skills exceed " + f"[AgentSkillExtractor] {len(existing_skill_records)} existing skills exceed " # noqa: G004 f"max_skills_in_prompt={max_skills_in_prompt}, selecting top-k" ) with timed("select_top_k_skills"): @@ -846,7 +846,7 @@ async def extract_and_save( prompt_template = self._select_prompt(new_case_records) logger.debug( - f"[AgentSkillExtractor] Incremental extraction: cluster={cluster_id}, " + f"[AgentSkillExtractor] Incremental extraction: cluster={cluster_id}, " # noqa: G004 f"new_cases={len(new_case_records)}, existing_skills={len(existing_skill_records)}" ) @@ -856,7 +856,7 @@ async def extract_and_save( ) if not llm_result: logger.warning( - f"[AgentSkillExtractor] LLM extraction failed for cluster={cluster_id}" + f"[AgentSkillExtractor] LLM extraction failed for cluster={cluster_id}" # noqa: G004 ) return empty_result @@ -894,12 +894,12 @@ async def extract_and_save( index = int(op.get("index", -1)) except (ValueError, TypeError): logger.warning( - f"[AgentSkillExtractor] update index is not a valid integer: {op.get('index')!r}, skipping" + f"[AgentSkillExtractor] update index is not a valid integer: {op.get('index')!r}, skipping" # noqa: G004 ) continue if index in processed_indices: logger.warning( - f"[AgentSkillExtractor] Duplicate operation on index {index}, skipping update" + f"[AgentSkillExtractor] Duplicate operation on index {index}, skipping update" # noqa: G004 ) continue processed_indices.add(index) @@ -925,16 +925,16 @@ async def extract_and_save( elif action == "none": logger.debug( - f"[AgentSkillExtractor] No-op for cluster={cluster_id}" + f"[AgentSkillExtractor] No-op for cluster={cluster_id}" # noqa: G004 ) else: logger.warning( - f"[AgentSkillExtractor] Unknown action '{action}', skipping" + f"[AgentSkillExtractor] Unknown action '{action}', skipping" # noqa: G004 ) logger.info( - f"[AgentSkillExtractor] cluster={cluster_id} operations applied: " + f"[AgentSkillExtractor] cluster={cluster_id} operations applied: " # noqa: G004 f"added={len(result.added_records)}, updated={update_count}, " f"deleted={len(result.deleted_ids)}. note: {update_note}" ) diff --git a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py index b223ac53..b8d57d14 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/atomic_fact_extractor.py @@ -75,10 +75,10 @@ def _parse_timestamp(self, timestamp) -> datetime: # Try parsing ISO format return datetime.fromisoformat(timestamp.replace('Z', '+00:00')) except (ValueError, AttributeError): - logger.error(f"Failed to parse timestamp: {timestamp}") + logger.error(f"Failed to parse timestamp: {timestamp}") # noqa: G004 return get_now_with_timezone() else: - logger.error(f"Unknown timestamp format: {timestamp}") + logger.error(f"Unknown timestamp format: {timestamp}") # noqa: G004 return get_now_with_timezone() def _format_timestamp(self, dt: datetime) -> str: @@ -155,7 +155,7 @@ def _parse_llm_response(self, response: str) -> Dict[str, Any]: pass # 5. If all fail, raise exception - logger.error(f"Unable to parse LLM response: {response[:200]}...") + logger.error(f"Unable to parse LLM response: {response[:200]}...") # noqa: G004 raise ValueError("Unable to parse LLM response into valid JSON format") async def _extract_atomic_fact( @@ -238,7 +238,7 @@ async def _extract_atomic_fact( ) logger.debug( - f"Successfully extracted atomic fact, containing {len(atomic_fact_obj.atomic_fact)} atomic facts (embeddings generated)" + f"Successfully extracted atomic fact, containing {len(atomic_fact_obj.atomic_fact)} atomic facts (embeddings generated)" # noqa: G004 ) return atomic_fact_obj @@ -267,9 +267,9 @@ async def extract_atomic_fact( return await self._extract_atomic_fact( input_text, timestamp, user_id=user_id, group_id=group_id ) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( - f"Retrying to extract atomic fact {retry + 1}/5: {e}" + f"Retrying to extract atomic fact {retry + 1}/5: {e}" # noqa: G004 ) if retry == 4: logger.error("Failed to extract atomic fact after 5 retries") diff --git a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py index 194ef8d9..8facfb92 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/episode_memory_extractor.py @@ -106,11 +106,11 @@ def _parse_timestamp(self, timestamp) -> datetime: return datetime.fromisoformat(timestamp.replace('Z', '+00:00')) except (ValueError, AttributeError): # Fallback to current time if parsing fails - logger.error(f"Failed to parse timestamp: {timestamp}") + logger.error(f"Failed to parse timestamp: {timestamp}") # noqa: G004 return get_now_with_timezone() else: # Unknown format, fallback to current time - logger.error(f"Failed to parse timestamp: {timestamp}") + logger.error(f"Failed to parse timestamp: {timestamp}") # noqa: G004 return get_now_with_timezone() def _format_timestamp(self, dt: datetime) -> str: @@ -203,7 +203,7 @@ async def _extract_episode( EpisodeMemory (contains episode field) """ logger.debug( - f"📚 Starting Episode extraction, use_group_prompt={use_group_prompt}" + f"📚 Starting Episode extraction, use_group_prompt={use_group_prompt}" # noqa: G004 ) memcell = request.memcell @@ -296,8 +296,8 @@ async def _extract_episode( # Validation passed, exit retry loop break - except Exception as e: - logger.warning(f"Episode extraction retry {i + 1}/5: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Episode extraction retry {i + 1}/5: {e}") # noqa: G004 if i == 4: raise Exception( "Episode memory extraction failed after 5 retries" @@ -337,7 +337,7 @@ async def _extract_episode( extend=embedding_data, # Add embedding to extend field ) - logger.debug(f"✅ Episode extraction completed: subject='{title}'") + logger.debug(f"✅ Episode extraction completed: subject='{title}'") # noqa: G004 return episode_memory async def extract_memory( @@ -366,7 +366,7 @@ async def extract_memory( is_group_episode = request.user_id is None logger.debug( - f"[extract_memory] Extracting {'group' if is_group_episode else 'personal'} Episode, " + f"[extract_memory] Extracting {'group' if is_group_episode else 'personal'} Episode, " # noqa: G004 f"user_id={request.user_id}, group_id={request.group_id}" ) @@ -398,6 +398,6 @@ async def _compute_embedding(self, text: str) -> Optional[dict]: "embedding": vec.tolist() if hasattr(vec, "tolist") else list(vec), "vector_model": vs.get_model_name(), # Use unified get_model_name() method } - except Exception as e: - logger.error(f"Episode Embedding computation failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Episode Embedding computation failed: {e}") # noqa: G004 return None diff --git a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py index 9b41e09c..54cb9a57 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/foresight_extractor.py @@ -97,11 +97,11 @@ async def generate_foresights_for_conversation( try: if retry == 0: logger.info( - f"🎯 Generating foresight associations for conversation: user_id={user_id}" + f"🎯 Generating foresight associations for conversation: user_id={user_id}" # noqa: G004 ) else: logger.info( - f"🎯 Generating foresight associations for conversation: user_id={user_id}, retry {retry}/5" + f"🎯 Generating foresight associations for conversation: user_id={user_id}, retry {retry}/5" # noqa: G004 ) # Build prompt (static prompt template via PromptManager) @@ -114,13 +114,13 @@ async def generate_foresights_for_conversation( # Call LLM to generate associations logger.debug( - f"📝 Starting LLM call to generate foresight associations, prompt length: {len(prompt)}" + f"📝 Starting LLM call to generate foresight associations, prompt length: {len(prompt)}" # noqa: G004 ) response = await self.llm_provider.generate( prompt=prompt, temperature=0.3 ) logger.debug( - f"✅ LLM call completed, response length: {len(response) if response else 0}" + f"✅ LLM call completed, response length: {len(response) if response else 0}" # noqa: G004 ) # Parse JSON response @@ -142,19 +142,19 @@ async def generate_foresights_for_conversation( foresights = foresights[:10] elif len(foresights) < 4: logger.warning( - f"Generated foresight associations less than 4, actual count: {len(foresights)}" + f"Generated foresight associations less than 4, actual count: {len(foresights)}" # noqa: G004 ) logger.info( - f"✅ Successfully generated {len(foresights)} foresight associations" + f"✅ Successfully generated {len(foresights)} foresight associations" # noqa: G004 ) for i, memory in enumerate(foresights[:3], 1): - logger.info(f" Association {i}: {memory.foresight}") + logger.info(f" Association {i}: {memory.foresight}") # noqa: G004 return foresights - except Exception as e: - logger.warning(f"Foresight generation retry {retry + 1}/5: {e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Foresight generation retry {retry + 1}/5: {e}") # noqa: G004 if retry == 4: logger.error("Foresight generation failed after 5 retries") return [] @@ -183,7 +183,7 @@ def _clean_date_string(date_str: Optional[str]) -> Optional[str]: # Validate format is YYYY-MM-DD if not re.match(r'^\d{4}-\d{2}-\d{2}$', cleaned): logger.warning( - f"Invalid time format, does not match YYYY-MM-DD: original='{date_str}', cleaned='{cleaned}'" + f"Invalid time format, does not match YYYY-MM-DD: original='{date_str}', cleaned='{cleaned}'" # noqa: G004 ) return None @@ -194,7 +194,7 @@ def _clean_date_string(date_str: Optional[str]) -> Optional[str]: datetime(year, month, day) return cleaned except ValueError as e: - logger.warning(f"Invalid date value: '{cleaned}', error: {e}") + logger.warning(f"Invalid date value: '{cleaned}', error: {e}") # noqa: G004 return None async def _parse_foresights_response( @@ -308,15 +308,15 @@ async def _parse_foresights_response( return foresights else: - logger.error(f"Response is not in JSON array format: {data}") + logger.error(f"Response is not in JSON array format: {data}") # noqa: G004 return [] except json.JSONDecodeError as e: - logger.error(f"Error parsing JSON response: {e}") - logger.debug(f"Response content: {response[:200]}...") + logger.error(f"Error parsing JSON response: {e}") # noqa: G004 + logger.debug(f"Response content: {response[:200]}...") # noqa: G004 return [] - except Exception as e: - logger.error(f"Error parsing foresight response: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error parsing foresight response: {e}") # noqa: G004 return [] def _extract_start_time_from_timestamp(self, timestamp: datetime) -> str: @@ -353,8 +353,8 @@ def _calculate_end_time_from_duration( return end_date.strftime('%Y-%m-%d') - except Exception as e: - logger.error(f"Error calculating end time from duration: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error calculating end time from duration: {e}") # noqa: G004 return None def _calculate_duration_days(self, start_time: str, end_time: str) -> Optional[int]: @@ -378,6 +378,6 @@ def _calculate_duration_days(self, start_time: str, end_time: str) -> Optional[i duration = end_date - start_date return duration.days - except Exception as e: - logger.error(f"Error calculating duration: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"Error calculating duration: {e}") # noqa: G004 return None diff --git a/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py b/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py index 8a0f0fd2..77b62c8a 100644 --- a/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py +++ b/methods/EverCore/src/memory_layer/memory_extractor/profile_extractor.py @@ -158,7 +158,7 @@ async def extract_memory( # Initialize profile if old_profile is None: logger.info( - f"[ProfileExtractor] No old_profile for user={request.user_id}, creating new" + f"[ProfileExtractor] No old_profile for user={request.user_id}, creating new" # noqa: G004 ) current_profile = ProfileMemory( memory_type=MemoryType.PROFILE, @@ -168,7 +168,7 @@ async def extract_memory( ) else: logger.info( - f"[ProfileExtractor] Using old_profile for user={request.user_id}: " + f"[ProfileExtractor] Using old_profile for user={request.user_id}: " # noqa: G004 f"explicit={len(old_profile.explicit_info)}, implicit={len(old_profile.implicit_traits)}" ) current_profile = old_profile @@ -176,7 +176,7 @@ async def extract_memory( # Check if already processed ep_id = new_episode.get("id") if ep_id in current_profile.processed_episode_ids: - logger.info(f"Episode {ep_id} already processed, skipping") + logger.info(f"Episode {ep_id} already processed, skipping") # noqa: G004 return current_profile # Create ID mapping @@ -187,7 +187,7 @@ async def extract_memory( ) id_map = _create_id_mapping(all_ids) - logger.info(f"Processing profile: cluster={len(cluster_episodes)}, new=1") + logger.info(f"Processing profile: cluster={len(cluster_episodes)}, new=1") # noqa: G004 # Resolve target_user_name for TEAM scene target_user_name = request.target_user_name @@ -230,7 +230,7 @@ async def extract_memory( if current_profile.total_items() > compact_threshold: logger.info( - f"Profile has {current_profile.total_items()} items (threshold={compact_threshold}), " + f"Profile has {current_profile.total_items()} items (threshold={compact_threshold}), " # noqa: G004 f"compacting to {compact_target}..." ) current_profile = await self._compact_profile( @@ -306,12 +306,12 @@ async def _llm_update_profile( if op_type == ProfileItemType.EXPLICIT_INFO: explicit_list.append(data) logger.info( - f"[Profile] Added explicit_info: {data.get('description', '')[:30]}..." + f"[Profile] Added explicit_info: {data.get('description', '')[:30]}..." # noqa: G004 ) elif op_type == ProfileItemType.IMPLICIT_TRAITS: implicit_list.append(data) logger.info( - f"[Profile] Added implicit_trait: {data.get('trait', '')}..." + f"[Profile] Added implicit_trait: {data.get('trait', '')}..." # noqa: G004 ) elif action == ProfileAction.UPDATE: @@ -336,7 +336,7 @@ async def _llm_update_profile( ) else: target_list[index][key] = val - logger.info(f"[Profile] Updated {op_type}[{index}]") + logger.info(f"[Profile] Updated {op_type}[{index}]") # noqa: G004 elif action == ProfileAction.DELETE: op_type = op.get("type") @@ -350,7 +350,7 @@ async def _llm_update_profile( if 0 <= index < len(target_list) and reason: target_list.pop(index) logger.warning( - f"[Profile] Deleted {op_type}[{index}]: {reason}" + f"[Profile] Deleted {op_type}[{index}]: {reason}" # noqa: G004 ) result_dict = { @@ -359,8 +359,8 @@ async def _llm_update_profile( } return _replace_sources(result_dict, id_map, reverse=True) - except Exception as e: - logger.error(f"LLM update profile failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"LLM update profile failed: {e}") # noqa: G004 return None def _build_timestamp_map( @@ -455,8 +455,8 @@ async def _compact_profile( return profile - except Exception as e: - logger.error(f"LLM compact profile failed: {e}") + except Exception as e: # noqa: BLE001 + logger.error(f"LLM compact profile failed: {e}") # noqa: G004 return profile def _format_profile_for_llm(self, profile_dict: Dict[str, Any]) -> str: @@ -545,7 +545,7 @@ def _resolve_user_name( return name # Fallback to user_id itself logger.warning( - f"Could not resolve sender_name for user_id={user_id}, using user_id as fallback" + f"Could not resolve sender_name for user_id={user_id}, using user_id as fallback" # noqa: G004 ) return user_id @@ -564,7 +564,7 @@ def _parse_profile_response(self, response: str) -> Optional[Dict[str, Any]]: if brace_start >= 0 and brace_end > brace_start: try: data = json.loads(response[brace_start:brace_end]) - except Exception: + except Exception: # noqa: BLE001 logger.warning("Failed to parse profile response JSON") return None else: @@ -573,6 +573,6 @@ def _parse_profile_response(self, response: str) -> Optional[Dict[str, Any]]: update_note = data.get("update_note") or data.get("compact_note") if update_note: - logger.info(f"Profile update: {update_note}") + logger.info(f"Profile update: {update_note}") # noqa: G004 return data diff --git a/methods/EverCore/src/memory_layer/memory_manager.py b/methods/EverCore/src/memory_layer/memory_manager.py index 4c97a71a..7983d8fc 100644 --- a/methods/EverCore/src/memory_layer/memory_manager.py +++ b/methods/EverCore/src/memory_layer/memory_manager.py @@ -142,9 +142,9 @@ def _build_providers_mapping(self) -> None: continue try: self.providers_mapping[scene] = self._build_scene_provider(scene, cfg) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( - f"[MemoryManager] Failed to build provider for " + f"[MemoryManager] Failed to build provider for " # noqa: G004 f"scene '{scene}': {e}, falling back to default" ) @@ -183,7 +183,7 @@ async def extract_memcell( """ now = time.time() - logger.debug(f"[MemoryManager] Starting boundary detection (flush={flush})") + logger.debug(f"[MemoryManager] Starting boundary detection (flush={flush})") # noqa: G004 request = ConversationMemCellExtractRequest( history_raw_data_list, @@ -212,7 +212,7 @@ async def extract_memcell( return [], status_result logger.info( - f"[MemoryManager] ✅ {len(memcells)} MemCell(s) created, " + f"[MemoryManager] ✅ {len(memcells)} MemCell(s) created, " # noqa: G004 f"elapsed time: {time.time() - now:.2f} seconds" ) @@ -284,7 +284,7 @@ async def extract_memory( case _: logger.warning( - f"[MemoryManager] Unknown memory_type: {memory_type}" + f"[MemoryManager] Unknown memory_type: {memory_type}" # noqa: G004 ) status = 'error' return None @@ -333,7 +333,7 @@ async def _extract_episode( # Call extractor's extract_memory method # It will automatically determine whether to extract group or personal Episode based on user_id logger.debug( - f"[MemoryManager] Extracting {'group' if user_id is None else 'personal'} Episode: user_id={user_id}" + f"[MemoryManager] Extracting {'group' if user_id is None else 'personal'} Episode: user_id={user_id}" # noqa: G004 ) return await self._episode_extractor.extract_memory(request) @@ -414,7 +414,7 @@ async def _extract_atomic_fact( uid = user_id gid = group_id - logger.debug(f"[MemoryManager] Extracting AtomicFact: user_id={uid}") + logger.debug(f"[MemoryManager] Extracting AtomicFact: user_id={uid}") # noqa: G004 extractor = AtomicFactExtractor( llm_provider=self._get_provider_for_scene("extraction") diff --git a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py index 35987792..6f72d89e 100644 --- a/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py +++ b/methods/EverCore/src/memory_layer/profile_indexer/profile_indexer.py @@ -216,7 +216,7 @@ async def index_profile( return stats except Exception as e: - logger.error( + logger.error( # noqa: G201 "[ProfileIndexer] ❌ Failed to index profile: user_id=%s, group_id=%s, error=%s", user_id, group_id, @@ -247,7 +247,7 @@ async def delete_profile_index(self, user_id: str, group_id: str) -> int: deleted_count, ) return deleted_count - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("[ProfileIndexer] Failed to delete profile index: %s", e) return 0 @@ -274,7 +274,7 @@ async def _generate_embeddings(self, texts: List[str]) -> List[List[float]]: return vectors - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("[ProfileIndexer] Failed to generate embeddings: %s", e) return [] @@ -300,7 +300,7 @@ async def index_user_profile( try: indexer = get_bean_by_type(ProfileIndexer) return await indexer.index_profile(user_id, group_id, profile, doc_id=doc_id) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error("[ProfileIndexer] Failed to get indexer service: %s", e) return { "deleted_count": 0, diff --git a/methods/EverCore/src/memory_layer/profile_manager/manager.py b/methods/EverCore/src/memory_layer/profile_manager/manager.py index 6e6bf465..e025a0c8 100644 --- a/methods/EverCore/src/memory_layer/profile_manager/manager.py +++ b/methods/EverCore/src/memory_layer/profile_manager/manager.py @@ -116,7 +116,7 @@ async def extract_profiles( # Convert old_profiles list to dict by user_id old_profiles_dict: Dict[str, ProfileMemory] = {} - logger.info(f"[Profile] Processing {len(old_profiles or [])} old profiles") + logger.info(f"[Profile] Processing {len(old_profiles or [])} old profiles") # noqa: G004 for p in old_profiles or []: uid = ( p.get("user_id") if isinstance(p, dict) else getattr(p, "user_id", None) @@ -124,24 +124,24 @@ async def extract_profiles( p_dict = p if isinstance(p, dict) else p.to_dict() has_explicit = "explicit_info" in p_dict logger.info( - f"[Profile] Old profile: user_id={uid}, has_explicit_info={has_explicit}, keys={list(p_dict.keys())[:5]}" + f"[Profile] Old profile: user_id={uid}, has_explicit_info={has_explicit}, keys={list(p_dict.keys())[:5]}" # noqa: G004 ) if uid and has_explicit: old_profiles_dict[uid] = ProfileMemory.from_dict(p_dict) logger.info( - f"[Profile] Loaded profile for {uid}: {old_profiles_dict[uid].total_items()} items" + f"[Profile] Loaded profile for {uid}: {old_profiles_dict[uid].total_items()} items" # noqa: G004 ) results: List[ProfileMemory] = [] logger.info( - f"[Profile] user_id_list={user_id_list}, old_profiles_dict keys={list(old_profiles_dict.keys())}" + f"[Profile] user_id_list={user_id_list}, old_profiles_dict keys={list(old_profiles_dict.keys())}" # noqa: G004 ) # Extract for each user for user_id in user_id_list: old_profile = old_profiles_dict.get(user_id) logger.info( - f"[Profile] Looking for user_id={user_id}, found={old_profile is not None}" + f"[Profile] Looking for user_id={user_id}, found={old_profile is not None}" # noqa: G004 ) # --- Per-user original_data filtering (Layer 2 of 2) --- @@ -178,7 +178,7 @@ async def extract_profiles( for attempt in range(self.config.max_retries): try: logger.info( - f"Extracting profile for user {user_id} (attempt {attempt + 1})..." + f"Extracting profile for user {user_id} (attempt {attempt + 1})..." # noqa: G004 ) result = await self._extractor.extract_memory(request) @@ -186,27 +186,27 @@ async def extract_profiles( if result: self._stats["successful_extractions"] += 1 logger.info( - f"Profile extracted for {user_id}: {result.total_items()} items " + f"Profile extracted for {user_id}: {result.total_items()} items " # noqa: G004 f"(explicit: {len(result.explicit_info)}, implicit: {len(result.implicit_traits)})" ) results.append(result) else: logger.warning( - f"Profile extraction returned None for {user_id}" + f"Profile extraction returned None for {user_id}" # noqa: G004 ) if old_profile: results.append(old_profile) break - except Exception as e: + except Exception as e: # noqa: BLE001 logger.warning( - f"Profile extraction attempt {attempt + 1} for {user_id} failed: {e}" + f"Profile extraction attempt {attempt + 1} for {user_id} failed: {e}" # noqa: G004 ) if attempt < self.config.max_retries - 1: await asyncio.sleep(0.5 * (attempt + 1)) else: logger.error( - f"All profile extraction attempts failed for {user_id}" + f"All profile extraction attempts failed for {user_id}" # noqa: G004 ) if old_profile: results.append(old_profile) diff --git a/methods/EverCore/src/service/group_service.py b/methods/EverCore/src/service/group_service.py index 34c7557e..c3df6b68 100644 --- a/methods/EverCore/src/service/group_service.py +++ b/methods/EverCore/src/service/group_service.py @@ -160,7 +160,7 @@ async def ensure_group_exists( await repo.upsert_by_group_id(group_id, update_data) logger.debug("Group auto-registered: group_id=%s", group_id) - except Exception as e: + except Exception as e: # noqa: BLE001 # Fire-and-forget: log error but don't raise logger.warning( "Failed to auto-register group: group_id=%s, error=%s", group_id, e diff --git a/methods/EverCore/src/service/memcell_delete_service.py b/methods/EverCore/src/service/memcell_delete_service.py index 6ac8a6a1..443a1988 100644 --- a/methods/EverCore/src/service/memcell_delete_service.py +++ b/methods/EverCore/src/service/memcell_delete_service.py @@ -134,7 +134,7 @@ async def delete_by_id(self, memory_id: str) -> DeleteResult: return result except Exception as e: - logger.error( + logger.error( # noqa: G201 "Failed to delete by memory_id=%s: error=%s", memory_id, e, @@ -182,7 +182,7 @@ async def delete_by_filters( return result except Exception as e: - logger.error("Failed to delete by filters: error=%s", e, exc_info=True) + logger.error("Failed to delete by filters: error=%s", e, exc_info=True) # noqa: G201 raise # ------------------------------------------------------------------ diff --git a/methods/EverCore/src/service/raw_message_service.py b/methods/EverCore/src/service/raw_message_service.py index 0e3993c1..1d27c4bc 100644 --- a/methods/EverCore/src/service/raw_message_service.py +++ b/methods/EverCore/src/service/raw_message_service.py @@ -94,7 +94,7 @@ async def save_raw_messages( ) if message_id: saved_message_ids.append(message_id) - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to save RawData to RawMessage: data_id=%s, error=%s", raw_data.data_id, @@ -195,7 +195,7 @@ async def check_duplicate_message( ) return True return False - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to check duplicate message: group_id=%s, sender_id=%s, message_id=%s, error=%s", group_id, @@ -269,7 +269,7 @@ async def get_pending_raw_messages( len(results), ) return results - except Exception as e: + except Exception as e: # noqa: BLE001 logger.error( "Failed to get pending raw messages: sender_id=%s, group_ids=%s, error=%s", sender_id, diff --git a/methods/EverCore/src/service/sender_service.py b/methods/EverCore/src/service/sender_service.py index 2d771f44..aee04b9e 100644 --- a/methods/EverCore/src/service/sender_service.py +++ b/methods/EverCore/src/service/sender_service.py @@ -153,7 +153,7 @@ async def ensure_sender_exists( await repo.upsert_by_sender_id(sender_id, update_data) logger.debug("Sender auto-registered: sender_id=%s", sender_id) - except Exception as e: + except Exception as e: # noqa: BLE001 # Fire-and-forget: log error but don't raise logger.warning( "Failed to auto-register sender: sender_id=%s, error=%s", sender_id, e diff --git a/methods/EverCore/src/service/session_service.py b/methods/EverCore/src/service/session_service.py index 15e31307..8a0e1f4d 100644 --- a/methods/EverCore/src/service/session_service.py +++ b/methods/EverCore/src/service/session_service.py @@ -62,7 +62,7 @@ async def ensure_session_exists( await repo.upsert_by_session_id(session_id, update_data) logger.debug("Session auto-registered: session_id=%s", session_id) - except Exception as e: + except Exception as e: # noqa: BLE001 # Fire-and-forget: log error but don't raise logger.warning( "Failed to auto-register session: session_id=%s, error=%s", From b03386d3df2a639cd8d5591249c3ef7c30def91f Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:52:18 +0800 Subject: [PATCH 20/24] =?UTF-8?q?test(EverCore):=20Phase=202=20T2.6=20?= =?UTF-8?q?=E2=80=94=20performance=20benchmark=20scaffolding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds opt-in performance benchmarks for the two hot paths called out in roadmap §3.2: hybrid retrieval and end-to-end memorize. Per roadmap guidance, benchmarks are NOT wired into CI yet — the first ~3 months are baseline collection only, with thresholds added later. - tests/benchmarks/test_retrieve_hybrid_benchmark.py: measures MemoryManager.retrieve_mem_hybrid; tied to the retrieve_p95_latency SLO (target p95 < 500 ms). - tests/benchmarks/test_memorize_benchmark.py: measures end-to-end memorize for a single message. - Both skip cleanly when RUN_BENCHMARKS is unset or when the DI container / DTOs can't be imported — collection-only is safe in CI. - Adds optional `bench` dependency group (pytest-benchmark). Default `uv sync` does not pull it in; activate with `uv sync --group bench`. - `make benchmark` target sets RUN_BENCHMARKS=1 and prints the min/mean/median/p95/max columns. --- methods/EverCore/Makefile | 10 ++- methods/EverCore/pyproject.toml | 7 ++ methods/EverCore/pytest.ini | 1 + methods/EverCore/tests/benchmarks/README.md | 52 +++++++++++++ methods/EverCore/tests/benchmarks/__init__.py | 0 methods/EverCore/tests/benchmarks/conftest.py | 39 ++++++++++ .../benchmarks/test_memorize_benchmark.py | 64 ++++++++++++++++ .../test_retrieve_hybrid_benchmark.py | 75 +++++++++++++++++++ 8 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 methods/EverCore/tests/benchmarks/README.md create mode 100644 methods/EverCore/tests/benchmarks/__init__.py create mode 100644 methods/EverCore/tests/benchmarks/conftest.py create mode 100644 methods/EverCore/tests/benchmarks/test_memorize_benchmark.py create mode 100644 methods/EverCore/tests/benchmarks/test_retrieve_hybrid_benchmark.py diff --git a/methods/EverCore/Makefile b/methods/EverCore/Makefile index c0499896..ad7d48cc 100644 --- a/methods/EverCore/Makefile +++ b/methods/EverCore/Makefile @@ -1,4 +1,4 @@ -.PHONY: dev-setup setup-hooks lint ruff ruff-fix typecheck typecheck-pyright test test-unit test-integration test-e2e test-cov clean help +.PHONY: dev-setup setup-hooks lint ruff ruff-fix typecheck typecheck-pyright test test-unit test-integration test-e2e test-cov benchmark clean help # Default target help: @@ -15,6 +15,7 @@ help: @echo " test-integration - Run integration tests (needs docker-compose up)" @echo " test-e2e - Run end-to-end tests" @echo " test-cov - Run unit tests with coverage report (xml + term)" + @echo " benchmark - Run perf benchmarks (needs bench group + docker)" @echo " clean - Clean up generated files" @echo " help - Show this help message" @@ -93,6 +94,13 @@ test-cov: PYTHONPATH=src uv run pytest tests/ -m unit \ --cov=src --cov-report=xml --cov-report=term +# Performance benchmarks. Requires the optional `bench` group +# (`uv sync --group bench`) and RUN_BENCHMARKS=1 plus a running +# docker-compose stack. See tests/benchmarks/README.md. +benchmark: + PYTHONPATH=src RUN_BENCHMARKS=1 uv run pytest tests/benchmarks/ \ + --benchmark-only --benchmark-columns=min,mean,median,p95,max + # Clean up clean: find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true diff --git a/methods/EverCore/pyproject.toml b/methods/EverCore/pyproject.toml index 9bcfb9a3..9927adde 100644 --- a/methods/EverCore/pyproject.toml +++ b/methods/EverCore/pyproject.toml @@ -232,6 +232,13 @@ otel = [ "opentelemetry-instrumentation-pymongo>=0.48b0", ] +# Performance benchmarks. Opt-in; not pulled in by `uv sync` without `--group bench`. +# Install when running tests/benchmarks/. CI does not run benchmarks per +# roadmap §3.2 T2.6 (establish baseline first, add thresholds later). +bench = [ + "pytest-benchmark>=4.0.0", +] + # Full development environment - includes dev tools dev-full = [ {include-group = "dev"}, diff --git a/methods/EverCore/pytest.ini b/methods/EverCore/pytest.ini index 5ad216e3..7bc803b2 100644 --- a/methods/EverCore/pytest.ini +++ b/methods/EverCore/pytest.ini @@ -29,6 +29,7 @@ markers = performance: 标记为性能测试 smoke: 标记为冒烟测试 e2e: end-to-end tests covering full request flows + benchmark: performance benchmarks (opt-in via RUN_BENCHMARKS env) # 异步测试配置 asyncio_mode = auto diff --git a/methods/EverCore/tests/benchmarks/README.md b/methods/EverCore/tests/benchmarks/README.md new file mode 100644 index 00000000..88c5cb02 --- /dev/null +++ b/methods/EverCore/tests/benchmarks/README.md @@ -0,0 +1,52 @@ +# EverCore Performance Benchmarks + +Performance baselines for EverCore's hot paths. Phase 2 T2.6 of the +[code-quality roadmap](../../docs/dev_docs/code_quality_roadmap.md). + +## Scope + +Two paths are tracked: + +| Benchmark | Target | Roadmap citation | +|---|---|---| +| `retrieve_mem_hybrid` p50/p95 over 100 calls | `MemoryManager.retrieve_mem_hybrid` | §3.2 T2.6 | +| End-to-end memorize latency for a single message | `biz_layer.mem_memorize.memorize` | §3.2 T2.6 | + +## How to run + +Benchmarks are **local-only by default** — they require docker-compose +services (Redis, MongoDB, Elasticsearch, Milvus, LLM provider). They are +explicitly **not** wired into CI yet; the roadmap calls for establishing +a baseline first and adding thresholds after roughly three months of +measurement. + +```bash +# Bring up infrastructure +docker compose up -d + +# Run benchmarks +make benchmark + +# Save a baseline snapshot +uv run pytest tests/benchmarks/ --benchmark-save=baseline + +# Compare against the saved baseline +uv run pytest tests/benchmarks/ --benchmark-compare=baseline +``` + +## Adding a benchmark + +- Put the test under `tests/benchmarks/`. +- Use the `@pytest.mark.benchmark` marker (already registered in + `pytest.ini`). +- Skip cleanly when prerequisites are absent — see existing files for + the pattern. + +## Why these specific paths + +`retrieve_mem_hybrid` is the highest-traffic read path and the source of +the `retrieve_p95_latency` SLO target +([slo_definitions.md §3.2](../../docs/dev_docs/slo_definitions.md#32-slo-2-retrieve_p95_latency)). +End-to-end memorize touches LLM extraction, vectorisation, and three +storage layers — a useful integration-shape signal that catches +regressions individual unit tests would miss. diff --git a/methods/EverCore/tests/benchmarks/__init__.py b/methods/EverCore/tests/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/methods/EverCore/tests/benchmarks/conftest.py b/methods/EverCore/tests/benchmarks/conftest.py new file mode 100644 index 00000000..c9b483c7 --- /dev/null +++ b/methods/EverCore/tests/benchmarks/conftest.py @@ -0,0 +1,39 @@ +"""Pytest configuration for the benchmark suite. + +Benchmarks live in this directory only. Auto-applies the `benchmark` +marker and skips collection entirely when ``pytest-benchmark`` is not +installed, so a developer with just the default dev group can still +collect the rest of the test tree without errors. +""" + +from __future__ import annotations + +import pytest + + +def pytest_collection_modifyitems(config, items): + """Tag every collected item in this directory with `benchmark`.""" + for item in items: + if "tests/benchmarks/" in str(item.fspath): + item.add_marker(pytest.mark.benchmark) + + +def pytest_configure(config): + """Warn loudly if pytest-benchmark is missing. + + We don't fail the whole session — other tests may still run — but + a missing plugin would cause the actual benchmarks to silently + behave as plain pytest functions. + """ + if config.getoption("--collect-only"): + return + try: + import pytest_benchmark # noqa: F401 + except ImportError: + config.issue_config_time_warning( + pytest.PytestConfigWarning( + "pytest-benchmark not installed; benchmark timings will not be " + "captured. Install with `uv sync --group bench`." + ), + stacklevel=2, + ) diff --git a/methods/EverCore/tests/benchmarks/test_memorize_benchmark.py b/methods/EverCore/tests/benchmarks/test_memorize_benchmark.py new file mode 100644 index 00000000..a4a2939d --- /dev/null +++ b/methods/EverCore/tests/benchmarks/test_memorize_benchmark.py @@ -0,0 +1,64 @@ +"""Benchmark: end-to-end memorize latency for a single message. + +Targets ``biz_layer.mem_memorize.memorize``. Tied to the +``memorize_success_rate`` SLO (target ≥ 99.9%) and to the +``memorize_duration_seconds`` Prometheus histogram. + +Run via ``make benchmark`` with infrastructure up. +""" + +from __future__ import annotations + +import os + +import pytest + + +pytestmark = pytest.mark.benchmark + + +@pytest.fixture(scope="session") +def memorize_callable(): + """Resolve the memorize entry point, skipping cleanly if unavailable.""" + if not os.environ.get("RUN_BENCHMARKS"): + pytest.skip("RUN_BENCHMARKS not set; opt-in flag for live benchmarks") + try: + from biz_layer.mem_memorize import memorize + except ImportError as exc: + pytest.skip(f"Cannot import memorize: {exc}") + return memorize + + +@pytest.fixture +def sample_memorize_request(): + """A single-message MemorizeRequest sized as a typical conversation turn.""" + try: + from api_specs.dtos.memorize import MemorizeRequest, MessageItem + except ImportError: + pytest.skip("MemorizeRequest DTO not importable") + return MemorizeRequest( + user_id="bench_user", + group_id="bench_group", + session_id="bench_session", + new_raw_data_list=[ + MessageItem( + message_id="bench_msg_1", + sender="bench_user", + sender_name="Bench User", + content="benchmark message body", + type="text", + ) + ], + ) + + +def test_memorize_end_to_end_latency( + benchmark, memorize_callable, sample_memorize_request +): + """Measure end-to-end memorize duration for one message.""" + import asyncio + + def _bench(): + return asyncio.run(memorize_callable(sample_memorize_request)) + + benchmark(_bench) diff --git a/methods/EverCore/tests/benchmarks/test_retrieve_hybrid_benchmark.py b/methods/EverCore/tests/benchmarks/test_retrieve_hybrid_benchmark.py new file mode 100644 index 00000000..2ac4e88f --- /dev/null +++ b/methods/EverCore/tests/benchmarks/test_retrieve_hybrid_benchmark.py @@ -0,0 +1,75 @@ +"""Benchmark: hybrid memory retrieval latency. + +Targets ``MemoryManager.retrieve_mem_hybrid``. Tied to the +``retrieve_p95_latency`` SLO (target p95 < 500 ms; see +``docs/dev_docs/slo_definitions.md``). + +This test is skipped automatically unless the required runtime +infrastructure (Redis, MongoDB, ES, Milvus, LLM provider) is up. +Run via ``make benchmark`` after ``docker compose up -d``. +""" + +from __future__ import annotations + +import os + +import pytest + + +pytestmark = pytest.mark.benchmark + + +@pytest.fixture(scope="session") +def memory_manager(): + """Resolve a real MemoryManager from the DI container. + + The DI container is constructed lazily via bootstrap; if the + container cannot be brought up (env, deps), the entire benchmark + is skipped rather than failing. + """ + if not os.environ.get("RUN_BENCHMARKS"): + pytest.skip("RUN_BENCHMARKS not set; opt-in flag for live benchmarks") + try: + from core.di.utils import get_bean_by_type + from agentic_layer.memory_manager import MemoryManager + except ImportError as exc: + pytest.skip(f"Cannot import MemoryManager: {exc}") + try: + return get_bean_by_type(MemoryManager) + except Exception as exc: # noqa: BLE001 + pytest.skip(f"DI container unavailable: {exc}") + + +@pytest.fixture +def sample_request(): + """Construct a representative RetrieveMemRequest. + + Kept minimal — real benchmark runs should override via parametrize + with realistic payload sizes and tenant ids. + """ + try: + from api_specs.dtos.retrieve import RetrieveMemRequest + except ImportError: + pytest.skip("RetrieveMemRequest DTO not importable") + return RetrieveMemRequest( + user_id="bench_user", + query="test query", + top_k=10, + ) + + +def test_retrieve_mem_hybrid_p50_p95(benchmark, memory_manager, sample_request): + """Measure retrieve_mem_hybrid over the default pytest-benchmark rounds. + + pytest-benchmark reports min/mean/median/p95 automatically. The + roadmap target is p95 < 500 ms; do not yet enforce — record first. + """ + import asyncio + + async def _run(): + return await memory_manager.retrieve_mem_hybrid(sample_request) + + def _bench(): + return asyncio.run(_run()) + + benchmark(_bench) From a2cd97387f1a6f18c189daab891e86b458ab784f Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:52:51 +0800 Subject: [PATCH 21/24] docs(EverCore): update code-quality roadmap status (T2.6 done) Records that T2.6 (benchmark scaffolding) landed and names T2.4 (unit-test backfill) as the next-up item gating Phase 3 W6/W7. --- .../docs/dev_docs/code_quality_roadmap.md | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md index 30e4983d..e1fe2679 100644 --- a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md +++ b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md @@ -14,14 +14,22 @@ | Phase | State | Notes | |---|---|---| | Phase 1: Observability | ✅ **Done** (2026-05-26) | P0 (T1.1 `/livez`+`/readyz`, T1.2 JSON logging, T1.3 typed error metrics) and P1 (T1.4 OpenTelemetry opt-in skeleton, T1.5 SLO doc) all merged. | -| Phase 2: Test foundation | 🟡 **Partial** (2026-05-26) | T2.1 pytest job in CI (non-blocking until tests are explicitly tagged), T2.2 coverage XML + artifact upload, T2.3 path-based test markers + `make test-unit/integration/e2e`, T2.5 typecheck in CI (continue-on-error). T2.4 unit-test backfill and T2.6 benchmark baseline still pending. | +| Phase 2: Test foundation | 🟡 **Partial** (2026-05-26) | T2.1 pytest job in CI (non-blocking until tests are explicitly tagged), T2.2 coverage XML + artifact upload, T2.3 path-based test markers + `make test-unit/integration/e2e`, T2.5 typecheck in CI (continue-on-error), T2.6 benchmark scaffolding (opt-in, `make benchmark`). **Still pending: T2.4 unit-test backfill on critical paths.** | | Phase 3: try-catch cleanup | 🟡 **Week 5 done** (2026-05-26) | W5 mechanical fixes merged: 20× `traceback.print_exc` → `logger.exception`, duplicate `RetryConfig` consolidated (longjob is canonical), `ruff G` + `BLE` rules enabled with baseline noqa markers. W6 (retry refactor) and W7 (catch-all + custom exceptions + large try-block splits) deferred — both require 24h staging soak per §4.2 and are unsafe to bundle into a doc-driven session. | -**Where this session stopped**: at the Phase 3 W5 / W6 boundary. The -remaining work — Type-A/B/E retry refactor, custom exception hierarchy, -splitting the 241-line try block — depends on test backfill (T2.4) being -real before changes can be validated, and per §4.2 each PR needs a 24h -staging soak. Pick up Phase 3 W6 after T2.4 lands. +**Where this session stopped**: T2.4 (unit-test backfill on critical +paths) and Phase 3 W6/W7. T2.4 is the natural next item — Phase 3 W6/W7 +both depend on real coverage of `biz_layer/mem_memorize.py`, +`agentic_layer/memory_manager.py`, and +`memory_layer/memory_extractor/agent_skill_extractor.py` before the +retry refactor and custom exception hierarchy can be validated. Per §4.2 +each W6/W7 PR also needs a 24h staging soak. + +Order of attack from here: +1. T2.4 — unit-test backfill (3-5 days). +2. Phase 3 W6 — retry refactor (tenacity / circuit breaker) once T2.4 lands. +3. Phase 3 W7 — custom exception hierarchy + catch-all overhaul + + splitting the 241-line try block. --- From dc6bed71ad88f8ce1d4085f9196af7cfb14c8b50 Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 21:53:13 +0800 Subject: [PATCH 22/24] chore(EverCore): lock pytest-benchmark and transitive deps for the bench group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picked up by uv during T2.6 verification. No behavior change for the default sync — these only activate under `uv sync --group bench`. --- methods/EverCore/uv.lock | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/methods/EverCore/uv.lock b/methods/EverCore/uv.lock index 9d1205c3..a719186d 100644 --- a/methods/EverCore/uv.lock +++ b/methods/EverCore/uv.lock @@ -1534,6 +1534,9 @@ dependencies = [ ] [package.dev-dependencies] +bench = [ + { name = "pytest-benchmark" }, +] dev = [ { name = "black" }, { name = "ipdb" }, @@ -1647,6 +1650,7 @@ requires-dist = [ ] [package.metadata.requires-dev] +bench = [{ name = "pytest-benchmark", specifier = ">=4.0.0" }] cpu = [] dev = [ { name = "black", specifier = ">=25.1.0" }, @@ -2371,6 +2375,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "py-spy" version = "0.4.1" @@ -2594,6 +2607,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" }, +] + [[package]] name = "pytest-cov" version = "7.0.0" From a02190e6e5f05127ba2b16ea049719d11fb486bf Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 22:01:53 +0800 Subject: [PATCH 23/24] =?UTF-8?q?test(EverCore):=20Phase=202=20T2.4=20?= =?UTF-8?q?=E2=80=94=20unit-test=20backfill=20for=20agentic=20and=20biz=20?= =?UTF-8?q?layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes Phase 2 T2.4 of the code-quality roadmap. agent_skill_extractor was already covered by 193 existing tests, so the gap was in two modules: - tests/test_agentic_memory_manager.py (14 tests) Pins the swallow-and-return-empty contract on retrieve_mem_hybrid / retrieve_mem so Phase 3 W7's intentional change is a deliberate, reviewed diff. Also covers _classify_retrieve_error for the most common failure modes. Surfaces one xfail: the audit-flagged possibly-unbound bug where retrieve_mem(None) enters the fallback path but crashes inside QueryMetadata.from_request(None). Strict-xfail so Phase 3 W7's fix flips it naturally to xpass. - tests/test_biz_mem_memorize.py (15 tests) Covers if_memorize, _is_agent_case_quality_sufficient, _should_skip_atomic_fact_for_agent, _clone_episodes_for_users, and the _save_agent_case catch-all (returns 0 on save / conversion failure). Pure-logic tests; no DB or LLM fixtures required. Both files pass `ruff check`, `ty check`, and run cleanly under `pytest -m unit` without docker. --- .../docs/dev_docs/code_quality_roadmap.md | 23 +- .../tests/test_agentic_memory_manager.py | 291 ++++++++++++++++++ .../EverCore/tests/test_biz_mem_memorize.py | 276 +++++++++++++++++ 3 files changed, 578 insertions(+), 12 deletions(-) create mode 100644 methods/EverCore/tests/test_agentic_memory_manager.py create mode 100644 methods/EverCore/tests/test_biz_mem_memorize.py diff --git a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md index e1fe2679..203328b1 100644 --- a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md +++ b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md @@ -14,22 +14,21 @@ | Phase | State | Notes | |---|---|---| | Phase 1: Observability | ✅ **Done** (2026-05-26) | P0 (T1.1 `/livez`+`/readyz`, T1.2 JSON logging, T1.3 typed error metrics) and P1 (T1.4 OpenTelemetry opt-in skeleton, T1.5 SLO doc) all merged. | -| Phase 2: Test foundation | 🟡 **Partial** (2026-05-26) | T2.1 pytest job in CI (non-blocking until tests are explicitly tagged), T2.2 coverage XML + artifact upload, T2.3 path-based test markers + `make test-unit/integration/e2e`, T2.5 typecheck in CI (continue-on-error), T2.6 benchmark scaffolding (opt-in, `make benchmark`). **Still pending: T2.4 unit-test backfill on critical paths.** | +| Phase 2: Test foundation | ✅ **Done** (2026-05-26) | T2.1 pytest job in CI (non-blocking until tests are explicitly tagged), T2.2 coverage XML + artifact upload, T2.3 path-based test markers + `make test-unit/integration/e2e`, T2.5 typecheck in CI (continue-on-error), T2.6 benchmark scaffolding (opt-in, `make benchmark`), T2.4 unit-test backfill for `agentic_layer/memory_manager.py` and `biz_layer/mem_memorize.py` (`agent_skill_extractor.py` was already covered by 193 existing tests). One xfail surfaced — the audit-flagged possibly-unbound bug in `retrieve_mem`'s fallback path, queued for Phase 3 W7. | | Phase 3: try-catch cleanup | 🟡 **Week 5 done** (2026-05-26) | W5 mechanical fixes merged: 20× `traceback.print_exc` → `logger.exception`, duplicate `RetryConfig` consolidated (longjob is canonical), `ruff G` + `BLE` rules enabled with baseline noqa markers. W6 (retry refactor) and W7 (catch-all + custom exceptions + large try-block splits) deferred — both require 24h staging soak per §4.2 and are unsafe to bundle into a doc-driven session. | -**Where this session stopped**: T2.4 (unit-test backfill on critical -paths) and Phase 3 W6/W7. T2.4 is the natural next item — Phase 3 W6/W7 -both depend on real coverage of `biz_layer/mem_memorize.py`, -`agentic_layer/memory_manager.py`, and -`memory_layer/memory_extractor/agent_skill_extractor.py` before the -retry refactor and custom exception hierarchy can be validated. Per §4.2 -each W6/W7 PR also needs a 24h staging soak. +**Where this session stopped**: at the Phase 3 W5 / W6 boundary. Both +W6 (retry refactor) and W7 (catch-all + custom exception hierarchy + +splitting the 241-line try block) require a 24h staging soak per PR per +§4.2 and are unsafe to bundle into a doc-driven session. Order of attack from here: -1. T2.4 — unit-test backfill (3-5 days). -2. Phase 3 W6 — retry refactor (tenacity / circuit breaker) once T2.4 lands. -3. Phase 3 W7 — custom exception hierarchy + catch-all overhaul + - splitting the 241-line try block. +1. Phase 3 W6 — retry refactor (tenacity / circuit breaker). +2. Phase 3 W7 — custom exception hierarchy + catch-all overhaul + + splitting the 241-line try block. The xfail in + `tests/test_agentic_memory_manager.py::test_no_request_returns_empty_response` + is the audit's named possibly-unbound bug; fixing it in W7 will + flip the xfail to xpass. --- diff --git a/methods/EverCore/tests/test_agentic_memory_manager.py b/methods/EverCore/tests/test_agentic_memory_manager.py new file mode 100644 index 00000000..160995b0 --- /dev/null +++ b/methods/EverCore/tests/test_agentic_memory_manager.py @@ -0,0 +1,291 @@ +"""Unit tests for ``agentic_layer.memory_manager.MemoryManager``. + +Phase 2 T2.4 of the code-quality roadmap. Targets the thin orchestration +layer that dispatches retrieval and writes — specifically the error +fallback paths that the audit flagged as catch-all anti-patterns +(possibly-unbound, swallow-and-return-empty). + +Mock strategy: patch the DI container so __init__ resolves to MagicMock +services. Override individual methods on the instance for path-specific +behavior. No real database, no real LLM. +""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from api_specs.dtos.memory import RetrieveMemRequest, RetrieveMemResponse +from api_specs.memory_models import MemoryType, RetrieveMethod + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def memory_manager(): + """Construct a MemoryManager with the DI lookups stubbed out. + + Returns a real MemoryManager instance — only the boundaries it owns + are mocked. Individual tests further override instance methods when + they want to control the search-side behavior. + """ + with patch( + "agentic_layer.memory_manager.get_bean_by_type", + return_value=MagicMock(), + ): + from agentic_layer.memory_manager import MemoryManager + + return MemoryManager() + + +def _make_request( + retrieve_method: RetrieveMethod = RetrieveMethod.HYBRID, + query: str = "what did the user say about coffee", + top_k: int = 5, + memory_types=None, +) -> RetrieveMemRequest: + return RetrieveMemRequest( + user_id="u_test", + group_ids=None, + memory_types=memory_types + if memory_types is not None + else [MemoryType.EPISODIC_MEMORY], + top_k=top_k, + query=query, + retrieve_method=retrieve_method, + ) + + +# =========================================================================== +# retrieve_mem_hybrid +# =========================================================================== + + +class TestRetrieveMemHybrid: + """Tests for MemoryManager.retrieve_mem_hybrid. + + The audit flagged this as a swallow-and-return-empty handler. The + contract is: never raise — degrade to an empty response instead. + These tests pin that contract so a future refactor that propagates + errors must be a deliberate, reviewed change. + """ + + @pytest.mark.asyncio + async def test_happy_path_returns_response(self, memory_manager): + """Hits flow through to _to_response and are returned.""" + hits = [{"id": "m1", "score": 0.9}, {"id": "m2", "score": 0.7}] + expected = MagicMock(spec=RetrieveMemResponse) + + memory_manager._search_hybrid = AsyncMock(return_value=hits) + memory_manager._to_response = AsyncMock(return_value=expected) + + result = await memory_manager.retrieve_mem_hybrid(_make_request()) + + assert result is expected + memory_manager._search_hybrid.assert_awaited_once() + memory_manager._to_response.assert_awaited_once() + # _to_response should have been called with the hits and the request. + called_hits, _called_req = memory_manager._to_response.await_args.args + assert called_hits == hits + + @pytest.mark.asyncio + async def test_search_failure_returns_empty_response(self, memory_manager): + """If the hybrid search raises, we fall back to an empty response. + + Contract: never propagate. This is one of the audit's named + catch-all sites — we keep the behavior under test until a + Phase 3 W7 refactor deliberately changes it. + """ + empty = MagicMock(spec=RetrieveMemResponse) + memory_manager._search_hybrid = AsyncMock(side_effect=ConnectionError("ES down")) + memory_manager._to_response = AsyncMock(return_value=empty) + + result = await memory_manager.retrieve_mem_hybrid(_make_request()) + + assert result is empty + # _to_response is still called, but with an empty hits list. + called_hits, _called_req = memory_manager._to_response.await_args.args + assert called_hits == [] + + @pytest.mark.asyncio + async def test_timeout_also_swallowed(self, memory_manager): + """Timeouts use the same fallback path — verifies the except is broad.""" + empty = MagicMock(spec=RetrieveMemResponse) + memory_manager._search_hybrid = AsyncMock(side_effect=asyncio.TimeoutError()) + memory_manager._to_response = AsyncMock(return_value=empty) + + result = await memory_manager.retrieve_mem_hybrid(_make_request()) + + assert result is empty + + @pytest.mark.asyncio + async def test_empty_memory_types_uses_unknown_label(self, memory_manager): + """When memory_types is empty, no crash — request still flows.""" + memory_manager._search_hybrid = AsyncMock(return_value=[]) + memory_manager._to_response = AsyncMock( + return_value=MagicMock(spec=RetrieveMemResponse) + ) + + req = _make_request(memory_types=[]) + # memory_types=[] would fail Pydantic validation? It's + # default_factory=list, so empty is OK as a default — but the + # request validator requires user_id or group_ids, which we + # provide. memory_type label code path uses 'unknown'. + await memory_manager.retrieve_mem_hybrid(req) + + memory_manager._search_hybrid.assert_awaited_once() + + +# =========================================================================== +# _classify_retrieve_error +# =========================================================================== + + +class TestClassifyRetrieveError: + """Tests for MemoryManager._classify_retrieve_error. + + Verifies the metric label produced for the most common failure + modes. Tied to the error_type label of `retrieve_errors_total` + (see slo_definitions.md and exception_handling_analysis.md). + """ + + def test_timeout_class_name(self, memory_manager): + label = memory_manager._classify_retrieve_error(asyncio.TimeoutError()) + assert label == "timeout" + + def test_connection_error(self, memory_manager): + label = memory_manager._classify_retrieve_error( + ConnectionError("backend unreachable") + ) + assert label == "connection_error" + + def test_generic_exception_uses_snake_case_class_name(self, memory_manager): + class WeirdError(Exception): + pass + + label = memory_manager._classify_retrieve_error(WeirdError("boom")) + # classify_exception falls through to snake-cased class name when + # no semantic category matches. + assert label == "weird_error" + + def test_validation_message_matches_category(self, memory_manager): + label = memory_manager._classify_retrieve_error( + ValueError("invalid query: empty") + ) + # "invalid" in message → validation_error category + assert label == "validation_error" + + +# =========================================================================== +# retrieve_mem dispatcher +# =========================================================================== + + +class TestRetrieveMemDispatcher: + """Tests for MemoryManager.retrieve_mem. + + This is the public entry point that fans out into keyword / vector / + hybrid / agentic search. The audit noted it also has a catch-all that + returns an empty response — the dispatcher must not raise. + """ + + @pytest.mark.asyncio + async def test_dispatches_to_hybrid(self, memory_manager): + expected = MagicMock(spec=RetrieveMemResponse) + memory_manager.retrieve_mem_hybrid = AsyncMock(return_value=expected) + memory_manager._get_pending_messages = AsyncMock(return_value=[]) + memory_manager._build_combined_response = MagicMock(return_value=expected) + + await memory_manager.retrieve_mem(_make_request(RetrieveMethod.HYBRID)) + + memory_manager.retrieve_mem_hybrid.assert_awaited_once() + + @pytest.mark.asyncio + async def test_dispatches_to_keyword(self, memory_manager): + memory_manager.retrieve_mem_keyword = AsyncMock( + return_value=MagicMock(spec=RetrieveMemResponse) + ) + memory_manager._get_pending_messages = AsyncMock(return_value=[]) + memory_manager._build_combined_response = MagicMock( + return_value=MagicMock(spec=RetrieveMemResponse) + ) + + await memory_manager.retrieve_mem(_make_request(RetrieveMethod.KEYWORD)) + + memory_manager.retrieve_mem_keyword.assert_awaited_once() + + @pytest.mark.xfail( + reason=( + "Audit-flagged bug: when retrieve_mem_request is None, the " + "ValueError is caught but the fallback path then calls " + "QueryMetadata.from_request(None) which raises AttributeError. " + "Tracked in exception_handling_analysis.md; will be fixed in " + "Phase 3 W7. Once fixed, this test flips to xpass." + ), + strict=True, + ) + @pytest.mark.asyncio + async def test_no_request_returns_empty_response(self, memory_manager): + """Passing a falsy request should yield an empty response, not crash.""" + result = await memory_manager.retrieve_mem(None) + + assert isinstance(result, RetrieveMemResponse) + assert result.memories == [] + assert result.total_count == 0 + + @pytest.mark.asyncio + async def test_downstream_failure_does_not_propagate(self, memory_manager): + """If the chosen retrieval method raises, we return an empty response. + + Contract pin — same audit category as retrieve_mem_hybrid. + """ + memory_manager.retrieve_mem_hybrid = AsyncMock( + side_effect=RuntimeError("rerank crashed") + ) + memory_manager._get_pending_messages = AsyncMock(return_value=[]) + + result = await memory_manager.retrieve_mem(_make_request(RetrieveMethod.HYBRID)) + + assert isinstance(result, RetrieveMemResponse) + assert result.memories == [] + + +# =========================================================================== +# memorize delegation +# =========================================================================== + + +class TestMemorize: + """Tests for MemoryManager.memorize. + + This is a thin pass-through to biz_layer.mem_memorize.memorize. The + test pins the contract that exceptions are NOT swallowed here — that + happens deeper in biz_layer. Refactors should be deliberate. + """ + + @pytest.mark.asyncio + async def test_delegates_to_biz_layer(self, memory_manager): + with patch( + "agentic_layer.memory_manager.memorize", + new=AsyncMock(return_value=7), + ) as mock_memorize: + req = MagicMock() + count = await memory_manager.memorize(req) + + assert count == 7 + mock_memorize.assert_awaited_once_with(req) + + @pytest.mark.asyncio + async def test_propagates_biz_layer_exception(self, memory_manager): + """No catch here — biz_layer owns the catch-all.""" + with patch( + "agentic_layer.memory_manager.memorize", + new=AsyncMock(side_effect=ValueError("bad request")), + ): + with pytest.raises(ValueError, match="bad request"): + await memory_manager.memorize(MagicMock()) diff --git a/methods/EverCore/tests/test_biz_mem_memorize.py b/methods/EverCore/tests/test_biz_mem_memorize.py new file mode 100644 index 00000000..6a9f03ee --- /dev/null +++ b/methods/EverCore/tests/test_biz_mem_memorize.py @@ -0,0 +1,276 @@ +"""Unit tests for ``biz_layer.mem_memorize`` helpers. + +Phase 2 T2.4 of the code-quality roadmap. The full ``memorize`` entry +point and ``process_memory_extraction`` orchestration require live +DB/LLM mocks that belong in integration suites. This file targets the +pure-logic helpers and the catch-all-returning-0 sites the audit +flagged, so the contracts are pinned before the Phase 3 W7 refactor. +""" + +from __future__ import annotations + +from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from api_specs.memory_types import AgentCase, EpisodeMemory, MemCell, RawDataType +from api_specs.memory_models import MemoryType +from biz_layer.mem_memorize import ( + _clone_episodes_for_users, + _is_agent_case_quality_sufficient, + _save_agent_case, + _should_skip_atomic_fact_for_agent, + if_memorize, +) +from biz_layer.memorize_config import MemorizeConfig + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_memcell( + *, + user_id_list=None, + original_data=None, + timestamp=None, + rd_type: RawDataType = RawDataType.CONVERSATION, +) -> MemCell: + return MemCell( + user_id_list=user_id_list or ["u1"], + original_data=original_data + or [{"message": {"role": "user", "content": "hello"}}], + timestamp=timestamp or datetime(2026, 1, 1, 0, 0, 0), + event_id="evt_1", + group_id="g1", + type=rd_type, + ) + + +def _make_agent_case(quality_score=0.8) -> AgentCase: + return AgentCase( + memory_type=MemoryType.AGENT_CASE, + user_id="u1", + timestamp=datetime(2026, 1, 1), + task_intent="ship a feature", + approach="1. plan\n2. write tests\n3. ship", + quality_score=quality_score, + vector=[0.1, 0.2, 0.3], + vector_model="text-embedding-3-small", + ) + + +# =========================================================================== +# if_memorize +# =========================================================================== + + +def test_if_memorize_always_true(): + """Currently a placeholder; pin so a future gate change is deliberate.""" + assert if_memorize(_make_memcell()) is True + + +# =========================================================================== +# _is_agent_case_quality_sufficient +# =========================================================================== + + +class TestAgentCaseQualityThreshold: + """Quality-score gating for skill extraction.""" + + def test_score_above_threshold(self): + config = MemorizeConfig(skill_min_quality_score=0.5) + assert _is_agent_case_quality_sufficient(_make_agent_case(0.8), config) is True + + def test_score_at_threshold(self): + config = MemorizeConfig(skill_min_quality_score=0.5) + # `score < threshold` is the rejection condition; equality passes. + assert _is_agent_case_quality_sufficient(_make_agent_case(0.5), config) is True + + def test_score_below_threshold(self): + config = MemorizeConfig(skill_min_quality_score=0.5) + assert _is_agent_case_quality_sufficient(_make_agent_case(0.4), config) is False + + def test_score_none_treated_as_insufficient(self): + """`None` score is the no-LLM-judgment case — must be skipped.""" + config = MemorizeConfig(skill_min_quality_score=0.2) + assert _is_agent_case_quality_sufficient(_make_agent_case(None), config) is False + + +# =========================================================================== +# _should_skip_atomic_fact_for_agent +# =========================================================================== + + +class TestShouldSkipAtomicFactForAgent: + """Atomic-fact skip rule for agent conversations. + + Skip when: at least one message has tool_calls or role='tool', AND + cumulative assistant non-tool-call text length >= 1000 characters. + """ + + def test_no_tool_calls_does_not_skip(self): + mc = _make_memcell( + original_data=[ + {"message": {"role": "user", "content": "hello"}}, + {"message": {"role": "assistant", "content": "a" * 5000}}, + ], + rd_type=RawDataType.AGENTCONVERSATION, + ) + assert _should_skip_atomic_fact_for_agent(mc) is False + + def test_tool_calls_with_short_assistant_does_not_skip(self): + mc = _make_memcell( + original_data=[ + {"message": {"role": "user", "content": "hi"}}, + { + "message": { + "role": "assistant", + "content": "ok", + "tool_calls": [{"name": "do"}], + } + }, + {"message": {"role": "assistant", "content": "done"}}, + ], + rd_type=RawDataType.AGENTCONVERSATION, + ) + # The tool-call-bearing assistant message is excluded from the + # length sum; only the bare assistant "done" counts (4 chars). + assert _should_skip_atomic_fact_for_agent(mc) is False + + def test_tool_calls_with_long_assistant_skips(self): + mc = _make_memcell( + original_data=[ + {"message": {"role": "user", "content": "hi"}}, + { + "message": { + "role": "assistant", + "content": "thinking", + "tool_calls": [{"name": "search"}], + } + }, + {"message": {"role": "tool", "content": "result"}}, + {"message": {"role": "assistant", "content": "x" * 1200}}, + ], + rd_type=RawDataType.AGENTCONVERSATION, + ) + assert _should_skip_atomic_fact_for_agent(mc) is True + + def test_tool_role_alone_counts_as_tool_calls(self): + """A bare `role=tool` message triggers the gating, even without + a `tool_calls` field on the assistant turn.""" + mc = _make_memcell( + original_data=[ + {"message": {"role": "tool", "content": "result"}}, + {"message": {"role": "assistant", "content": "y" * 1200}}, + ], + rd_type=RawDataType.AGENTCONVERSATION, + ) + assert _should_skip_atomic_fact_for_agent(mc) is True + + +# =========================================================================== +# _clone_episodes_for_users +# =========================================================================== + + +class TestCloneEpisodesForUsers: + """Replicate a single group Episode across participants.""" + + def _make_state(self, participants): + group_ep = EpisodeMemory( + memory_type=MemoryType.EPISODIC_MEMORY, + user_id="g1", + timestamp=datetime(2026, 1, 1), + subject="team", + summary="shipped X", + episode="The team shipped X.", + ) + return SimpleNamespace( + group_episode_memories=[group_ep], participants=participants + ) + + def test_clones_one_per_participant(self): + state = self._make_state(["alice", "bob", "carol"]) + clones = _clone_episodes_for_users(state) + assert len(clones) == 3 + assert {c.user_id for c in clones} == {"alice", "bob", "carol"} + + def test_user_name_set_to_user_id(self): + state = self._make_state(["alice"]) + clones = _clone_episodes_for_users(state) + assert clones[0].user_name == "alice" + + def test_episode_content_preserved_per_clone(self): + """Each clone shares the source episode body — only user_id/name differ.""" + state = self._make_state(["alice", "bob"]) + clones = _clone_episodes_for_users(state) + for clone in clones: + assert clone.episode == "The team shipped X." + assert clone.summary == "shipped X" + + +# =========================================================================== +# _save_agent_case (catch-all returning 0) +# =========================================================================== + + +class TestSaveAgentCase: + """Audit-flagged catch-all: save failure → return 0, never raise. + + Phase 3 W7 will replace the bare ``except Exception`` with a typed + hierarchy. Until then, the contract is pinned here so a refactor + that propagates errors is a deliberate, reviewed change. + """ + + def _make_state(self, agent_case=None): + return SimpleNamespace( + agent_case=agent_case or _make_agent_case(), + memcell=_make_memcell(), + current_time=datetime(2026, 1, 1), + request=SimpleNamespace(session_id="sess_1"), + ) + + @pytest.mark.asyncio + async def test_save_success_returns_one(self): + with ( + patch( + "biz_layer.mem_memorize._convert_agent_case_to_doc", + return_value=MagicMock(), + ), + patch( + "biz_layer.mem_memorize.save_memory_docs", + new=AsyncMock(return_value={}), + ), + ): + count = await _save_agent_case(self._make_state()) + assert count == 1 + + @pytest.mark.asyncio + async def test_save_failure_returns_zero(self): + """save_memory_docs failure must not propagate.""" + with ( + patch( + "biz_layer.mem_memorize._convert_agent_case_to_doc", + return_value=MagicMock(), + ), + patch( + "biz_layer.mem_memorize.save_memory_docs", + new=AsyncMock(side_effect=ConnectionError("ES down")), + ), + ): + count = await _save_agent_case(self._make_state()) + assert count == 0 + + @pytest.mark.asyncio + async def test_doc_conversion_failure_returns_zero(self): + """Failure earlier in the pipeline is also swallowed.""" + with patch( + "biz_layer.mem_memorize._convert_agent_case_to_doc", + side_effect=ValueError("bad agent case"), + ): + count = await _save_agent_case(self._make_state()) + assert count == 0 From d912ca30b4cbdc22477377c2b1e2d5d89bc419bc Mon Sep 17 00:00:00 2001 From: twocucao Date: Tue, 26 May 2026 22:08:01 +0800 Subject: [PATCH 24/24] =?UTF-8?q?feat(EverCore):=20Phase=203=20W7=20founda?= =?UTF-8?q?tion=20=E2=80=94=20exception=20hierarchy=20+=20bug=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two W7-subset items that are safely landable without staging soak: 1. core/errors/ — typed exception hierarchy. EverCoreError is the common root so controllers can install a single except EverCoreError fallback while pattern-matching subtypes higher in the chain. Subclasses cover: - MemorizeError / MemoryCellPersistFailed — write-path failures - ExtractionError / LLMOutputFormatError — LLM-side failures, with LLMOutputFormatError carved out for the Type-B "feedback retry" pattern (parse failure feeds into next prompt) - RetrieveError / SearchBackendUnavailable — read-path failures - ProviderUnavailable — root-level (both paths can hit a provider outage), carries an explicit `provider` field for metric/label attribution Every class carries `tenant_id`, `cause`, and arbitrary `context` kwargs so structured logging and Sentry grouping work without parsing message strings. Purely additive — no callers migrated. 15 tests pin the hierarchy shape and attribute contract. 2. Fix audit-flagged possibly-unbound bug in MemoryManager.retrieve_mem. When the request was None, the validation ValueError was caught by the broad fallback, but the fallback then crashed inside QueryMetadata.from_request(None). Short-circuited None at the top so the swallow-and-empty contract is honored. The xfail in tests/test_agentic_memory_manager.py::test_no_request_returns_empty_response flips to xpass and is no longer marked. Roadmap §0 status updated. W6 retry refactor remains deferred — openai_provider._execute_with_retry's key-rotation + status-code dispatch + per-error metric labels aren't a tenacity drop-in and need behavior-equivalence tests plus 24h staging soak per §4.2. --- .../docs/dev_docs/code_quality_roadmap.md | 21 ++- .../src/agentic_layer/memory_manager.py | 27 ++- methods/EverCore/src/core/errors/__init__.py | 51 ++++++ methods/EverCore/src/core/errors/business.py | 159 ++++++++++++++++++ .../tests/test_agentic_memory_manager.py | 19 +-- methods/EverCore/tests/test_core_errors.py | 129 ++++++++++++++ 6 files changed, 384 insertions(+), 22 deletions(-) create mode 100644 methods/EverCore/src/core/errors/__init__.py create mode 100644 methods/EverCore/src/core/errors/business.py create mode 100644 methods/EverCore/tests/test_core_errors.py diff --git a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md index 203328b1..a5efdb0d 100644 --- a/methods/EverCore/docs/dev_docs/code_quality_roadmap.md +++ b/methods/EverCore/docs/dev_docs/code_quality_roadmap.md @@ -15,7 +15,7 @@ |---|---|---| | Phase 1: Observability | ✅ **Done** (2026-05-26) | P0 (T1.1 `/livez`+`/readyz`, T1.2 JSON logging, T1.3 typed error metrics) and P1 (T1.4 OpenTelemetry opt-in skeleton, T1.5 SLO doc) all merged. | | Phase 2: Test foundation | ✅ **Done** (2026-05-26) | T2.1 pytest job in CI (non-blocking until tests are explicitly tagged), T2.2 coverage XML + artifact upload, T2.3 path-based test markers + `make test-unit/integration/e2e`, T2.5 typecheck in CI (continue-on-error), T2.6 benchmark scaffolding (opt-in, `make benchmark`), T2.4 unit-test backfill for `agentic_layer/memory_manager.py` and `biz_layer/mem_memorize.py` (`agent_skill_extractor.py` was already covered by 193 existing tests). One xfail surfaced — the audit-flagged possibly-unbound bug in `retrieve_mem`'s fallback path, queued for Phase 3 W7. | -| Phase 3: try-catch cleanup | 🟡 **Week 5 done** (2026-05-26) | W5 mechanical fixes merged: 20× `traceback.print_exc` → `logger.exception`, duplicate `RetryConfig` consolidated (longjob is canonical), `ruff G` + `BLE` rules enabled with baseline noqa markers. W6 (retry refactor) and W7 (catch-all + custom exceptions + large try-block splits) deferred — both require 24h staging soak per §4.2 and are unsafe to bundle into a doc-driven session. | +| Phase 3: try-catch cleanup | 🟡 **W5 done + W7 foundation** (2026-05-26) | W5 mechanical fixes merged: 20× `traceback.print_exc` → `logger.exception`, duplicate `RetryConfig` consolidated (longjob is canonical), `ruff G` + `BLE` rules enabled with baseline noqa markers. W7 foundation: `core/errors/` exception hierarchy (`EverCoreError` root + `MemorizeError`, `ExtractionError`, `RetrieveError`, `ProviderUnavailable`, etc.) — purely additive, no callers migrated yet. Possibly-unbound bug in `retrieve_mem(None)` fixed; the xfail flipped to xpass. **Still deferred**: W6 retry refactor (Type A — `openai_provider._execute_with_retry` carries key-rotation/status-code branching that isn't a tenacity drop-in; Types B/E unsurveyed), and W7 catch-all migration. Both need 24h staging soak per §4.2. | **Where this session stopped**: at the Phase 3 W5 / W6 boundary. Both W6 (retry refactor) and W7 (catch-all + custom exception hierarchy + @@ -23,12 +23,19 @@ splitting the 241-line try block) require a 24h staging soak per PR per §4.2 and are unsafe to bundle into a doc-driven session. Order of attack from here: -1. Phase 3 W6 — retry refactor (tenacity / circuit breaker). -2. Phase 3 W7 — custom exception hierarchy + catch-all overhaul + - splitting the 241-line try block. The xfail in - `tests/test_agentic_memory_manager.py::test_no_request_returns_empty_response` - is the audit's named possibly-unbound bug; fixing it in W7 will - flip the xfail to xpass. +1. Phase 3 W6 — retry refactor. Type-A `openai_provider._execute_with_retry` + is more complex than a tenacity drop-in: key rotation + status-code + dispatch + per-error metric labels mean the migration needs careful + behavior-equivalence tests before a 24h staging soak. Simpler Type-A + sites (e.g. `agentic_layer/rerank_deepinfra.py:104`) are good + starter candidates. +2. Phase 3 W7 — migrate the business-layer catch-alls to raise the + typed exceptions defined in `core.errors`, with controller-side + `except EverCoreError` boundaries. The foundation classes already + exist; the per-site migration is the staging-soak work. +3. Phase 3 W7 — split the 241-line try block in + `biz_layer/mem_memorize.py::memorize` once the surrounding catch + site is replaced with typed boundaries. --- diff --git a/methods/EverCore/src/agentic_layer/memory_manager.py b/methods/EverCore/src/agentic_layer/memory_manager.py index ee56d980..cf1d0657 100644 --- a/methods/EverCore/src/agentic_layer/memory_manager.py +++ b/methods/EverCore/src/agentic_layer/memory_manager.py @@ -202,11 +202,30 @@ async def retrieve_mem( Returns: RetrieveMemResponse containing retrieval results """ - try: - # Validate request parameters - if not retrieve_mem_request: - raise ValueError("retrieve_mem_request is required for retrieve_mem") + # Short-circuit a missing request before the try block. The fallback + # path below dereferences ``retrieve_mem_request`` (for QueryMetadata + # and Metadata) — raising inside the try with None would crash the + # except handler itself with AttributeError. Return the empty shape + # directly so the swallow-and-empty contract is preserved. + if not retrieve_mem_request: + logger.error( + "retrieve_mem called with no request; returning empty response" + ) + return RetrieveMemResponse( + profiles=[], + memories=[], + total_count=0, + has_more=False, + query_metadata=QueryMetadata(), + metadata=Metadata( + source="retrieve_mem_service", + user_id="", + memory_types=[], + ), + pending_messages=[], + ) + try: # Get memory types from request (defaults already applied in converter) memory_types = retrieve_mem_request.memory_types diff --git a/methods/EverCore/src/core/errors/__init__.py b/methods/EverCore/src/core/errors/__init__.py new file mode 100644 index 00000000..a704abed --- /dev/null +++ b/methods/EverCore/src/core/errors/__init__.py @@ -0,0 +1,51 @@ +"""EverCore custom exception hierarchy. + +Phase 3 W7 of the code-quality roadmap. The audit +([`docs/dev_docs/exception_handling_analysis.md`](../../../docs/dev_docs/exception_handling_analysis.md)) +identified ~71% of try blocks as bare ``except Exception``. The cleanup +plan is to replace business-layer catch-alls with typed exceptions +propagated upward, then catch the specific types at controller +boundaries. + +This module is the foundation. It is **purely additive** — no existing +call sites are migrated here. Migration happens in follow-up PRs that +each carry their own test coverage and staging soak. + +Naming convention: +- ``EverCoreError`` — the common root. Every typed exception in this + hierarchy derives from it so controllers can install a single + fallback ``except EverCoreError`` while still pattern-matching + specific subtypes higher in the chain. +- ``*Error`` — stage / domain bucket (``MemorizeError``, + ``RetrieveError``, ``ExtractionError``). +- ``*Failed`` — a specific failure mode within a bucket + (``MemoryCellPersistFailed``, ``ProviderUnavailable``). + +Each error carries a ``tenant_id`` so structured logging can attach the +right scope without an additional logger.bind call. Tenant-scoping is +optional — pass ``None`` when the failure happens before tenant context +is available. +""" + +from core.errors.business import ( + EverCoreError, + ExtractionError, + LLMOutputFormatError, + MemorizeError, + MemoryCellPersistFailed, + ProviderUnavailable, + RetrieveError, + SearchBackendUnavailable, +) + + +__all__ = [ + "EverCoreError", + "ExtractionError", + "LLMOutputFormatError", + "MemorizeError", + "MemoryCellPersistFailed", + "ProviderUnavailable", + "RetrieveError", + "SearchBackendUnavailable", +] diff --git a/methods/EverCore/src/core/errors/business.py b/methods/EverCore/src/core/errors/business.py new file mode 100644 index 00000000..28a3a80f --- /dev/null +++ b/methods/EverCore/src/core/errors/business.py @@ -0,0 +1,159 @@ +"""Business-layer exception hierarchy. + +See ``core.errors`` package docstring for context. This module defines +the concrete classes; ``core/errors/__init__.py`` re-exports them. + +Why a custom hierarchy at all? + +1. **Pattern match at the boundary, not deep inside.** Today the + business layer catches ``Exception``, logs a string, and returns an + empty result. The controller can't tell apart a transient backend + blip from a malformed request. With typed errors the controller can + choose: 503 for ``SearchBackendUnavailable``, 400 for + ``LLMOutputFormatError`` after retries exhaust, etc. +2. **Carry context across the catch boundary.** Plain ``Exception`` + loses tenant_id, stage, and request id once the message is + formatted. Subclasses preserve those fields as attributes so + structured logging and Sentry grouping work without parsing + message strings. +3. **Stable taxonomy for metrics.** The existing + ``record_*_error(error_type=...)`` helpers use exception class names + already (T1.3). Naming the buckets explicitly here means a refactor + of the underlying call chain doesn't accidentally rename the metric + label. +""" + +from __future__ import annotations + +from typing import Any, Optional + + +class EverCoreError(Exception): + """Common root for every typed EverCore failure. + + Controllers may install a single ``except EverCoreError`` as the + domain-level fallback. Subclasses provide finer-grained matching + higher in the chain when needed. + + Attributes: + tenant_id: Tenant under which the failure occurred. May be + ``None`` if the failure pre-dates tenant resolution (e.g. + during auth). + cause: Optional underlying exception preserved for logging. + Use ``raise X from y`` whenever possible; this attribute + exists for callers that catch then re-raise across an + async boundary where the ``__cause__`` chain would + otherwise be lost. + """ + + def __init__( + self, + message: str, + *, + tenant_id: Optional[str] = None, + cause: Optional[BaseException] = None, + **context: Any, + ) -> None: + super().__init__(message) + self.tenant_id = tenant_id + self.cause = cause + self.context = context + + def __repr__(self) -> str: + bits = [f"message={self.args[0]!r}"] + if self.tenant_id is not None: + bits.append(f"tenant_id={self.tenant_id!r}") + if self.context: + bits.append(f"context={self.context!r}") + return f"{type(self).__name__}({', '.join(bits)})" + + +# --------------------------------------------------------------------------- +# Write path — memorize +# --------------------------------------------------------------------------- + + +class MemorizeError(EverCoreError): + """Anything that prevents a memorize request from completing. + + Subclasses narrow the cause. Catching ``MemorizeError`` at the + controller boundary is the default; catch a subclass when the + response differs (e.g. retryable vs not). + """ + + +class MemoryCellPersistFailed(MemorizeError): + """A MemCell could not be saved to its target store. + + Distinguished from extraction failure: extraction succeeded, the + write side did not. Typically retryable. + """ + + +class ExtractionError(EverCoreError): + """LLM-driven extraction failed. + + Covers boundary detection, episode/atomic-fact/foresight + extraction, agent case/skill extraction. The retry policy for the + extraction layer (Phase 3 W6 Type-B "feedback retry") should + typically convert provider-side failures into this class before + propagating. + """ + + +class LLMOutputFormatError(ExtractionError): + """LLM responded but the output didn't match the expected schema. + + Distinguished from a provider-side failure: the call succeeded, + the *content* was wrong. The Type-B "feedback retry" pattern + catches this specifically so it can include the parsing error in + the next prompt; other failures don't get that treatment. + """ + + +# --------------------------------------------------------------------------- +# Read path — retrieve +# --------------------------------------------------------------------------- + + +class RetrieveError(EverCoreError): + """Anything that prevents a retrieve request from completing. + + Default controller behavior should be to return 503 with retry + guidance for ``SearchBackendUnavailable``, and 500 (logged loudly) + for any other ``RetrieveError`` since those indicate a logic bug + rather than a transient backend issue. + """ + + +class SearchBackendUnavailable(RetrieveError): + """Elasticsearch / Milvus / Redis is unreachable or returning errors. + + Retryable in principle. Today's catch-all returns an empty result + set; the Phase 3 W7 migration replaces that with this error so the + controller can pick the response code. + """ + + +class ProviderUnavailable(EverCoreError): + """An external LLM / embedding provider is unreachable or rate-limited. + + Carries a ``provider`` context field so logging / metrics can + attribute the failure correctly. Sits at the root (not under + ``RetrieveError`` or ``ExtractionError``) because both write and + read paths can hit the same condition. + """ + + def __init__( + self, + message: str, + *, + provider: Optional[str] = None, + tenant_id: Optional[str] = None, + cause: Optional[BaseException] = None, + **context: Any, + ) -> None: + super().__init__( + message, tenant_id=tenant_id, cause=cause, provider=provider, **context + ) + self.provider = provider diff --git a/methods/EverCore/tests/test_agentic_memory_manager.py b/methods/EverCore/tests/test_agentic_memory_manager.py index 160995b0..1c7d7408 100644 --- a/methods/EverCore/tests/test_agentic_memory_manager.py +++ b/methods/EverCore/tests/test_agentic_memory_manager.py @@ -219,19 +219,16 @@ async def test_dispatches_to_keyword(self, memory_manager): memory_manager.retrieve_mem_keyword.assert_awaited_once() - @pytest.mark.xfail( - reason=( - "Audit-flagged bug: when retrieve_mem_request is None, the " - "ValueError is caught but the fallback path then calls " - "QueryMetadata.from_request(None) which raises AttributeError. " - "Tracked in exception_handling_analysis.md; will be fixed in " - "Phase 3 W7. Once fixed, this test flips to xpass." - ), - strict=True, - ) @pytest.mark.asyncio async def test_no_request_returns_empty_response(self, memory_manager): - """Passing a falsy request should yield an empty response, not crash.""" + """Passing a falsy request yields an empty response, not a crash. + + Fixed in Phase 3 W7 — the original implementation entered the + broad fallback path which then crashed inside + ``QueryMetadata.from_request(None)`` (audit's named + possibly-unbound case). The fix short-circuits None at the top + of ``retrieve_mem`` so the response shape is honored. + """ result = await memory_manager.retrieve_mem(None) assert isinstance(result, RetrieveMemResponse) diff --git a/methods/EverCore/tests/test_core_errors.py b/methods/EverCore/tests/test_core_errors.py new file mode 100644 index 00000000..0f08094d --- /dev/null +++ b/methods/EverCore/tests/test_core_errors.py @@ -0,0 +1,129 @@ +"""Unit tests for the ``core.errors`` exception hierarchy. + +Phase 3 W7 (subset). Pins the public contract of the hierarchy so +later migration PRs can rely on it without rediscovering the shape. +""" + +from __future__ import annotations + +import pytest + +from core.errors import ( + EverCoreError, + ExtractionError, + LLMOutputFormatError, + MemorizeError, + MemoryCellPersistFailed, + ProviderUnavailable, + RetrieveError, + SearchBackendUnavailable, +) + + +class TestHierarchy: + """Every concrete class roots at EverCoreError so the controller-side + catch-all stays a single line.""" + + def test_every_error_subclasses_evercoreerror(self): + for cls in ( + MemorizeError, + MemoryCellPersistFailed, + ExtractionError, + LLMOutputFormatError, + RetrieveError, + SearchBackendUnavailable, + ProviderUnavailable, + ): + assert issubclass(cls, EverCoreError), cls.__name__ + + def test_memorize_subclass_layout(self): + assert issubclass(MemoryCellPersistFailed, MemorizeError) + assert not issubclass(MemorizeError, MemoryCellPersistFailed) + + def test_extraction_subclass_layout(self): + assert issubclass(LLMOutputFormatError, ExtractionError) + + def test_retrieve_subclass_layout(self): + assert issubclass(SearchBackendUnavailable, RetrieveError) + + def test_provider_unavailable_is_root_level(self): + """Sits directly under EverCoreError — not Memorize or Retrieve — + because both paths can hit a provider outage.""" + assert issubclass(ProviderUnavailable, EverCoreError) + assert not issubclass(ProviderUnavailable, RetrieveError) + assert not issubclass(ProviderUnavailable, MemorizeError) + + +class TestEverCoreErrorAttributes: + """The base preserves tenant_id, cause, and arbitrary context.""" + + def test_minimal_construction(self): + err = EverCoreError("boom") + assert str(err) == "boom" + assert err.tenant_id is None + assert err.cause is None + assert err.context == {} + + def test_tenant_id_preserved(self): + err = EverCoreError("boom", tenant_id="t_1") + assert err.tenant_id == "t_1" + + def test_cause_preserved_for_async_re_raise(self): + underlying = ValueError("orig") + err = EverCoreError("wrapped", cause=underlying) + assert err.cause is underlying + + def test_context_captures_arbitrary_kwargs(self): + err = EverCoreError("boom", stage="vectorize", attempt=3) + assert err.context == {"stage": "vectorize", "attempt": 3} + + def test_repr_includes_tenant_and_context(self): + err = EverCoreError("boom", tenant_id="t_1", stage="rerank") + rep = repr(err) + assert "tenant_id='t_1'" in rep + assert "stage" in rep + + +class TestProviderUnavailable: + """ProviderUnavailable adds a typed ``provider`` field.""" + + def test_provider_field(self): + err = ProviderUnavailable("429 from openai", provider="openai") + assert err.provider == "openai" + assert err.context["provider"] == "openai" + + def test_provider_optional(self): + err = ProviderUnavailable("upstream down") + assert err.provider is None + + def test_tenant_and_provider_together(self): + err = ProviderUnavailable( + "rate limited", provider="anthropic", tenant_id="t_42" + ) + assert err.provider == "anthropic" + assert err.tenant_id == "t_42" + + +class TestRaiseFromUsage: + """The hierarchy is meant to be used with ``raise X from y``. + + The chained exception remains accessible via ``__cause__`` so + logging and stack capture work the standard way. + """ + + def test_raise_from_preserves_chain(self): + try: + try: + raise ConnectionError("backend down") + except ConnectionError as exc: + raise SearchBackendUnavailable( + "elasticsearch unreachable", cause=exc + ) from exc + except SearchBackendUnavailable as caught: + assert isinstance(caught.__cause__, ConnectionError) + assert caught.cause is caught.__cause__ + + def test_pattern_match_at_root(self): + """Controllers can catch the root and still receive subclasses.""" + with pytest.raises(EverCoreError): + raise MemoryCellPersistFailed("mongo timeout")