From 3382c226163a83be64a05e10acd4e37e6603ad65 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 9 Jul 2026 03:30:55 +0000 Subject: [PATCH 1/5] feat(index): add optional Milvus derived index backend Signed-off-by: Cheney Zhang --- config.example.toml | 16 + docs/api.md | 15 +- docs/architecture.md | 26 +- docs/cascade_runbook.md | 16 +- docs/configuration.md | 22 + docs/knowledge.md | 18 +- docs/multimodal.md | 2 +- docs/overview.md | 6 +- docs/reflection.md | 11 +- docs/storage_layout.md | 43 +- src/everos/README.md | 2 +- src/everos/config/__init__.py | 7 +- src/everos/config/default.toml | 18 + src/everos/config/settings.py | 26 + src/everos/core/persistence/memory_root.py | 13 + .../infra/persistence/index/__init__.py | 329 ++++++++ .../infra/persistence/milvus/__init__.py | 49 ++ .../persistence/milvus/milvus_manager.py | 87 ++ src/everos/infra/persistence/milvus/repos.py | 119 +++ .../infra/persistence/milvus/repository.py | 750 ++++++++++++++++++ .../memory/cascade/handlers/agent_case.py | 2 +- .../memory/cascade/handlers/agent_skill.py | 2 +- .../memory/cascade/handlers/atomic_fact.py | 2 +- src/everos/memory/cascade/handlers/episode.py | 2 +- .../memory/cascade/handlers/foresight.py | 2 +- .../cascade/handlers/knowledge_topic.py | 2 +- .../memory/cascade/handlers/user_profile.py | 2 +- src/everos/memory/cascade/registry.py | 2 +- src/everos/memory/get/filters_adapter.py | 19 + src/everos/memory/get/manager.py | 12 +- src/everos/memory/search/filters.py | 215 ++++- src/everos/memory/search/manager.py | 7 +- src/everos/memory/search/recall/agent_case.py | 56 +- .../memory/search/recall/agent_skill.py | 57 +- .../memory/search/recall/atomic_fact.py | 39 +- src/everos/memory/search/recall/episode.py | 36 +- .../memory/search/recall/knowledge_topic.py | 39 +- src/everos/memory/search/recall/profile.py | 2 +- .../memory/strategies/extract_agent_skill.py | 4 +- .../memory/strategies/extract_user_profile.py | 2 +- .../memory/strategies/reflect_episodes.py | 2 +- src/everos/service/get.py | 6 +- tests/unit/test_config/test_settings.py | 27 + .../test_persistence/test_memory_root.py | 3 + .../unit/test_infra/test_milvus/test_repo.py | 127 +++ .../unit/test_memory/test_get/test_manager.py | 4 +- .../test_memory/test_search/test_filters.py | 33 + .../test_memory/test_search/test_manager.py | 16 +- .../test_search/test_recall_episode.py | 68 +- .../test_recall_knowledge_topic.py | 121 ++- uv.lock | 227 +++++- 51 files changed, 2312 insertions(+), 401 deletions(-) create mode 100644 src/everos/infra/persistence/index/__init__.py create mode 100644 src/everos/infra/persistence/milvus/__init__.py create mode 100644 src/everos/infra/persistence/milvus/milvus_manager.py create mode 100644 src/everos/infra/persistence/milvus/repos.py create mode 100644 src/everos/infra/persistence/milvus/repository.py create mode 100644 tests/unit/test_infra/test_milvus/test_repo.py diff --git a/config.example.toml b/config.example.toml index f8043127f..0145dd686 100644 --- a/config.example.toml +++ b/config.example.toml @@ -56,3 +56,19 @@ max_concurrent = 5 # # [lancedb] # read_consistency_seconds = 5.0 + +# ── Optional Milvus derived index ───────────────────── +# Markdown remains the source of truth. Set the index backend to Milvus +# when you want the rebuildable vector/BM25 index to live in Milvus. +# Leave uri empty for Milvus Lite at /.index/milvus/milvus.db. +# +# [index] +# backend = "milvus" +# +# [milvus] +# uri = "" # or "http://localhost:19530" +# token = "" # required for Zilliz Cloud/auth-enabled Milvus +# db_name = "" +# consistency_level = "Session" +# dimension = 1024 # must match your embedding model output +# collection_prefix = "everos" diff --git a/docs/api.md b/docs/api.md index 422b47ffa..d19361f4e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -93,8 +93,9 @@ bare FastAPI `detail`); see [Errors](#errors). `/add` and `/flush` write the markdown file (the source of truth) **synchronously** — when the call returns with `status: "extracted"`, -the new entry exists on disk. The LanceDB vector / BM25 / scalar index -is rebuilt by the in-process **cascade coroutine asynchronously**. +the new entry exists on disk. The configured vector / BM25 / scalar +index backend is rebuilt by the in-process **cascade coroutine +asynchronously**. That means `/search` and `/get` may not see a record immediately after the `/flush` that produced it. Typical sync latency is sub-second, but @@ -371,7 +372,7 @@ A recursive boolean tree of predicates. Used by `/search.filters` and `/get.filters`. The Pydantic envelope only checks the recursive combinator shape; field-level validity (which scalar fields are filterable, which operators apply, value coercion) runs when the -node is compiled to a LanceDB `where` clause server-side. Compile +node is compiled to a backend-specific filter clause server-side. Compile errors surface as `422` with the offending field / operator in `error.message`. @@ -464,12 +465,12 @@ Examples: |---|---| | `"keyword"` | BM25 only — pure lexical match, no embedding cost | | `"vector"` | Dense vector ANN only — semantic recall, no lexical | -| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter in a single LanceDB query | +| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter against the configured derived index backend | | `"agentic"` | Iterative cluster-path retrieval driven by a cross-encoder rerank loop; higher quality at higher latency / cost | `"hybrid"` is the default because it balances recall and precision -with one LanceDB roundtrip. `"agentic"` calls the LLM in a loop and -should be reserved for offline or background workflows. +without requiring the agentic loop. `"agentic"` calls the LLM in a loop +and should be reserved for offline or background workflows. ### GetMemoryType @@ -613,7 +614,7 @@ scope. this `(session_id, app_id, project_id)`, or it was already flushed). `/flush` is synchronous with respect to markdown persistence: by the -time the response returns, the new entry is on disk. LanceDB index +time the response returns, the new entry is on disk. Derived index sync is still asynchronous — see [Eventual consistency](#eventual-consistency). diff --git a/docs/architecture.md b/docs/architecture.md index afb5efe30..4aae7826a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ │ + reflection + strategies + get + events │ ├──────────────────────────────────────────────────────┤ │ infra/persistence (Storage adapters; infra/ may host other adapter types) │ -│ markdown + sqlite + lancedb │ +│ markdown + sqlite + derived index │ └──────────────────────────────────────────────────────┘ Cross-cutting (used by all layers, depends on none): @@ -68,8 +68,8 @@ layers = [ └────────────────────────────────────────────────────────────────┘ ┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ - │ Markdown │ │ SQLite │ │ LanceDB │ - │ (truth) │ │ (state) │ │ (index) │ + │ Markdown │ │ SQLite │ │ Derived index │ + │ (truth) │ │ (state) │ │ LanceDB/Milvus │ ├──────────────┤ ├──────────────┤ ├─────────────────┤ │ entries + │ │ change queue │ │ vector ANN │ │ frontmatter │ │ + state/LSN │ │ BM25 (Tantivy) │ @@ -78,7 +78,7 @@ layers = [ └──────────────┘ └──────────────┘ └─────────────────┘ │ │ │ ▼ ▼ ▼ - memory-root/ .index/sqlite/ .index/lancedb/ + memory-root/ .index/sqlite/ .index// (truth source) (system data) (rebuildable) ``` @@ -101,10 +101,13 @@ External message │ │ ▼ ▼ 4a. SQLite 4b. memory.cascade (async daemon) - audit watches md → diff entries → LanceDB sync + audit watches md → diff entries → index sync ``` -**Key guarantee**: md write is strongly consistent (fsync). LanceDB is eventually consistent. LanceDB unavailability does not block response — changes buffer in the SQLite `md_change_state` queue, replayed on recovery. +**Key guarantee**: md write is strongly consistent (fsync). The derived +index is eventually consistent. Index backend unavailability does not block +response — changes buffer in the SQLite `md_change_state` queue, replayed on +recovery. ## Read path @@ -115,8 +118,8 @@ User query 1. service.search │ ▼ -2. memory.search (hybrid) single LanceDB query = - BM25 + vector ANN + scalar filter +2. memory.search (hybrid) BM25 + vector ANN + scalar filter + through the configured index backend │ ▼ 3. (optional) read md original markdown for context @@ -139,12 +142,13 @@ extract/ ### `memory/cascade/` -Daemon that watches markdown changes and syncs to LanceDB: +Daemon that watches markdown changes and syncs to the configured derived +index backend: - inotify / FSEvents file watcher (cross-platform via `watchdog`) - 500ms debounce - Entry-level diff (added / changed / removed) -- LanceDB single-transaction update (text + vector columns atomic) +- Per-entry index upsert / delete (text + vector columns update together) - LSN-based crash recovery via the SQLite `md_change_state` queue - Handlers for all eight business kinds: episode, atomic_fact, foresight, user_profile, agent_case, agent_skill, knowledge_document, knowledge_topic @@ -225,7 +229,7 @@ holding **only memory extraction algorithms**: everalgo is: - **Stateless** — pure functions, no class hierarchy -- **No I/O** — does not touch md files / LanceDB / SQLite +- **No I/O** — does not touch md files, derived indexes, or SQLite - **No prompts inline** — extractors that accept a prompt-override parameter use the project-supplied value; others use their algo-bundled defaults This boundary lets everalgo be reused across product forms (this open-source build, EverOS Cloud, OpenClaw plugins, etc.). diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index 3e0dcd900..d836672f9 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -1,9 +1,13 @@ # Cascade Runbook -The cascade daemon keeps LanceDB in sync with the markdown files under -the memory root. Service / entry points only ever write markdown; the -daemon is the **sole** writer of the LanceDB index. This runbook covers -the recurring operational questions. +The cascade daemon keeps the configured derived index in sync with the +markdown files under the memory root. Service / entry points only ever write +markdown; the daemon is the **sole** writer of the derived index. This runbook +covers the recurring operational questions. + +Sections that mention LanceDB-specific schemas, index cache, file descriptors, +or `lance error` messages apply to the default LanceDB backend. Milvus uses the +same cascade queue with backend-specific collection management. ## What runs where @@ -13,7 +17,7 @@ providers in order: 1. **Metrics** — Prometheus collector. 2. **LLM** — LLM client initialisation. 3. **SQLite** — system DB + schema (`SQLModel.metadata.create_all`). -4. **LanceDB** — async connection + schema verification + FTS indexes. +4. **Derived index** — async connection + schema verification + search indexes. 5. **Cascade** — watcher + scanner + worker, all in-process tasks. 6. **OME** — offline memory engine. @@ -23,7 +27,7 @@ The cascade subsystem itself is three independent loops: |---|---|---| | Watcher | `watchdog` filesystem events (sync thread) | `md_change_state.upsert` per registered kind | | Scanner | Periodic walk (`scan_interval_seconds`, default 30 s) | Same — catches changes the watcher missed | -| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → LanceDB upsert / delete | +| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → index upsert / delete | Every loop talks to the same `md_change_state` sqlite table. The worker's claim mode (`pending → processing → done/failed`) keeps diff --git a/docs/configuration.md b/docs/configuration.md index 98fdf3479..6848d7839 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,26 @@ everos init --root /data/everos | `read_consistency_seconds` | float \| null | `null` | Read consistency interval. `null` = no check, `0` = strict, `>0` = eventual. | | `index_cache_size_bytes` | int | `16777216` | Upper bound on LanceDB index cache (16 MB default). | +### `[index]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `backend` | `"lancedb"` \| `"milvus"` | `"lancedb"` | Rebuildable vector/BM25 index backend used by cascade, search, and get. Markdown remains the source of truth; SQLite remains the system state store. | + +### `[milvus]` + +Milvus is optional. Install with `everos[milvus]`, then set +`[index].backend = "milvus"`. + +| Field | Type | Default | Description | +|---|---|---|---| +| `uri` | string \| null | `""` | Empty uses Milvus Lite at `/.index/milvus/milvus.db`. Set to a Milvus server or Zilliz Cloud endpoint to use an external service. | +| `token` | string \| null | `""` | Token for Zilliz Cloud or auth-enabled Milvus. | +| `db_name` | string \| null | `""` | Optional Milvus database name. | +| `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | +| `dimension` | int | `1024` | Dense vector dimension. Must match the configured embedding model output. | +| `collection_prefix` | string | `"everos"` | Prefix for EverOS Milvus collections. | + ### `[llm]` | Field | Type | Default | Required | Description | @@ -228,3 +248,5 @@ Examples: | `[llm] api_key = "sk-..."` | `EVEROS_LLM__API_KEY=sk-...` | | `[sqlite] busy_timeout_ms = 10000` | `EVEROS_SQLITE__BUSY_TIMEOUT_MS=10000` | | `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | +| `[index] backend = "milvus"` | `EVEROS_INDEX__BACKEND=milvus` | +| `[milvus] uri = "http://localhost:19530"` | `EVEROS_MILVUS__URI=http://localhost:19530` | diff --git a/docs/knowledge.md b/docs/knowledge.md index d3ad92dd3..a7d7b1770 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -48,8 +48,8 @@ Each level corresponds to a different granularity of API: ## Storage layout Every document is a self-contained directory. Markdown files are the -single source of truth; SQLite and LanceDB are derived indexes built -automatically by the cascade daemon. +single source of truth; SQLite and the configured vector/BM25 index are +derived indexes built automatically by the cascade daemon. ``` ~/.everos///knowledge/ @@ -73,16 +73,16 @@ automatically by the cascade daemon. ### Storage roles ``` -Markdown (source of truth) + SQLite (structured state) + LanceDB (vector + BM25 index) +Markdown (source of truth) + SQLite (structured state) + derived vector/BM25 index ``` | Store | What it holds | Role | |-------|---------------|------| | Markdown | Document metadata, summaries, topic content, original files | Single source of truth; human-readable and editable | | SQLite | Document rows, topic rows (with content), change queue | Structured queries, paginated lists, count aggregation | -| LanceDB | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | +| Derived index | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | -Even if SQLite and LanceDB data is corrupted, as long as the Markdown +Even if SQLite and derived index data is corrupted, as long as the Markdown files are intact, the indexes can be fully rebuilt via the cascade daemon. ### Markdown format @@ -500,7 +500,7 @@ query ─→ embed ─→ keyword (BM25) ─┐ ``` 1. **Embed** — the query is embedded using the configured embedding provider -2. **Recall** — dual-channel retrieval from LanceDB: +2. **Recall** — dual-channel retrieval from the configured derived index: - BM25 channel: keyword matching on `summary_tokens` + `content_tokens` - ANN channel: nearest-neighbor search on the `vector` column - In `hybrid` mode, both channels run in parallel @@ -531,7 +531,7 @@ export EVEROS_KNOWLEDGE__SEARCH__RERANK_N=100 ## Cascade sync The cascade daemon watches the knowledge Markdown directory for file -changes and keeps SQLite + LanceDB in sync. +changes and keeps SQLite + the configured derived index in sync. ``` md file written @@ -539,7 +539,7 @@ md file written → worker picks up from queue (≤1s poll interval) → handler dispatched by file type: index.md → KnowledgeDocumentHandler → SQLite upsert (metadata) - N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + LanceDB upsert + N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + index upsert ``` The topic handler uses a SHA-256 content digest to skip unchanged files — @@ -597,7 +597,7 @@ curl .../documents?app_id=tenant_b&project_id=proj_1 # → { "documents": [], "total": 0 } ``` -Storage paths, SQLite rows, and LanceDB indexes are all scoped by +Storage paths, SQLite rows, and derived index rows are all scoped by `app_id` + `project_id`. ## End-to-end walkthrough diff --git a/docs/multimodal.md b/docs/multimodal.md index eeb181811..5c6b1a6da 100644 --- a/docs/multimodal.md +++ b/docs/multimodal.md @@ -41,7 +41,7 @@ POST /api/v2/memory/add boundary detector → extraction LLM → memory cell (MemCell) │ ▼ - markdown (truth) + SQLite (state) + LanceDB (vector + BM25) + markdown (truth) + SQLite (state) + derived vector/BM25 index │ ▼ retrievable via /search and /get like any text memory diff --git a/docs/overview.md b/docs/overview.md index 07297673d..0bfda5b7d 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -11,7 +11,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory - Local deployment for personal agents or small teams - Conversation, workflow, agent-trace, file-knowledge → structured memory - Hybrid retrieval (BM25 + vector + scalar filter) -- Cascade index sync (md edit → LanceDB sub-second) +- Cascade index sync (md edit → derived index) - Dual-track memory (user-track / agent-track) - Offline memory evolution (Foresight / AtomicFact / Profile / Skill), including Reflection — a consolidation strategy within the OME that @@ -30,7 +30,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory ### 1. Markdown as Source of Truth ``` -delete all LanceDB / SQLite files → can rebuild from md +delete all derived index / SQLite files → can rebuild from md delete any md file → memory is gone ``` @@ -42,7 +42,7 @@ User trust comes from physical visibility — the user can `cat` / `vim` / `grep |---|---|---| | Markdown files | Truth source — entries, frontmatter | Search (grep is degraded fallback only) | | SQLite | Queue, cascade audit log, sensitive data isolation | Vector / full-text | -| LanceDB | Vector ANN + BM25 + scalar filter, single-query hybrid | Be the source of truth (loss = rebuild from md) | +| Derived index | Vector ANN + BM25 + scalar filter | Be the source of truth (loss = rebuild from md) | ### 3. Algorithm-orchestration separation diff --git a/docs/reflection.md b/docs/reflection.md index 63c09f1fd..f3e920bcc 100644 --- a/docs/reflection.md +++ b/docs/reflection.md @@ -147,14 +147,15 @@ narrative. ## Storage layout -Memory uses Markdown as the single source of truth; SQLite and LanceDB are -derived indexes built automatically by the cascade daemon. +Memory uses Markdown as the single source of truth; SQLite and the configured +vector/BM25 index backend are derived indexes built automatically by the +cascade daemon. | Store | What it holds | Role | |---|---|---| | Markdown | Episode bodies, merged narratives, archive markers | Single source of truth; human-readable and editable | | SQLite | Clusters and members, consolidation audit records | Structured state and queries | -| LanceDB | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | +| Derived index | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | The **merged narrative** is written to the Episode daily-log Markdown; its frontmatter marks that it came from a cluster: @@ -186,7 +187,7 @@ deprecated_entries: --- ``` -> Soft-archive, not delete: even if SQLite / LanceDB are corrupted, as long +> Soft-archive, not delete: even if SQLite / the derived index are corrupted, as long > as the Markdown is intact the indexes can be fully rebuilt — and every > consolidation remains traceable back to its original content. @@ -355,5 +356,5 @@ curl -s -X POST "$BASE/memory/search" \ ## See also - [how-memory-works.md](how-memory-works.md) — Episodes and the memory extraction pipeline -- [storage_layout.md](storage_layout.md) — Markdown + SQLite + LanceDB stack +- [storage_layout.md](storage_layout.md) — Markdown + SQLite + derived index stack - [api.md](api.md) — full HTTP API reference diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 860446648..71447905a 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -3,8 +3,9 @@ How `everos` lays out a memory-root on disk: directory tree, file naming, frontmatter chassis, and entry-id encoding. -The contents are the **source of truth**; SQLite and LanceDB are -derived indexes that can be rebuilt from markdown alone. +The contents are the **source of truth**; SQLite and the configured +vector/BM25 index backend are derived indexes that can be rebuilt from +markdown alone. ## 1. Memory-root tree @@ -49,8 +50,10 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) -│ └── lancedb/ -│ └── .lance/ one directory per LanceDB table +│ ├── lancedb/ +│ │ └── .lance/ default derived index backend +│ └── milvus/ +│ └── milvus.db Milvus Lite database when configured │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -64,7 +67,7 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the -runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the +runtime-required dirs (`.index/{sqlite,lancedb,milvus}/`, `.tmp/`); the user-visible dirs are *not* pre-created — they appear on first write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. @@ -168,9 +171,9 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers > **File-level seq, not global**: the same `ep_20260601_00000001` may > appear across two different `user_id`s (each user has its own daily file). > Cross-table joins must therefore key on **`(scope_id, entry_id)`** -> rather than `entry_id` alone — see SQLite/LanceDB tables that follow. +> rather than `entry_id` alone — see the derived index tables that follow. -## 5. SQLite + LanceDB derived indexes +## 5. SQLite + derived indexes ``` .index/ @@ -178,9 +181,10 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ └── system.db state / audit / cascade queue + buffer / LSN │ (system tables: md_change_state, memcell, │ unprocessed_buffer, conversation_status, cluster) -└── lancedb/ - └── .lance/ one Arrow table per business kind — the per-kind - rows (text / vector / tokens / metadata) live here +├── lancedb/ +│ └── .lance/ default derived index backend +└── milvus/ + └── milvus.db Milvus Lite database when configured ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -190,13 +194,14 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers per-kind business rows. `reflection_report` is the audit trail for Reflection merges (cluster_id, mode, source_members, merged_entry_id, status). -- **LanceDB** ([`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/)) - holds the per-kind business rows, keyed `_` (so - cross-table joins use `(owner_id, entry_id)`); each table's `Vector(N)` - dimension matches the embedding model output. - -Episode and AtomicFact LanceDB tables carry a `deprecated_by: str | None` -column. When an episode is superseded by a Reflection merge, +- The **derived index backend** holds the per-kind business rows, keyed + `_` (so cross-table joins use `(owner_id, entry_id)`). + LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled + as the same rebuildable index backend and uses `.index/milvus/milvus.db` by + default for Milvus Lite. + +Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. +When an episode is superseded by a Reflection merge, `deprecated_by` is set to the merged episode's entry_id. Search filters automatically exclude rows where `deprecated_by IS NOT NULL`. @@ -226,5 +231,5 @@ this primitive is **schema-agnostic** — field-level semantics - Code: - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) - - [`infra/persistence/{markdown,sqlite,lancedb}/`](../src/everos/infra/persistence/) - - [`memory/cascade/`](../src/everos/memory/cascade/) (md → LanceDB sync) + - [`infra/persistence/{markdown,sqlite,lancedb,milvus,index}/`](../src/everos/infra/persistence/) + - [`memory/cascade/`](../src/everos/memory/cascade/) (md → derived index sync) diff --git a/src/everos/README.md b/src/everos/README.md index 8cc1cef96..2be6caf25 100644 --- a/src/everos/README.md +++ b/src/everos/README.md @@ -10,7 +10,7 @@ everos/ ├── entrypoints/ Presentation: cli + api ├── service/ Application: use case orchestration ├── memory/ Domain: extract + search + cascade + prompt_slots + models -├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb} +├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb, milvus, index} ├── component/ Cross-cutting providers: llm / embedding / config / utils ├── core/ Runtime base: observability / lifespan / context └── config/ Data: Settings + default.toml + prompt_slots templates diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index adbd1e354..6ce3d9686 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -2,7 +2,8 @@ Public API: from everos.config import ( - Settings, MemorySettings, SqliteSettings, LanceDBSettings, + Settings, MemorySettings, SqliteSettings, IndexSettings, LanceDBSettings, + MilvusSettings, LLMSettings, EmbeddingSettings, RerankSettings, BoundaryDetectionSettings, CascadeSettings, load_settings, resolve_root, @@ -15,9 +16,11 @@ from .settings import BoundaryDetectionSettings as BoundaryDetectionSettings from .settings import CascadeSettings as CascadeSettings from .settings import EmbeddingSettings as EmbeddingSettings +from .settings import IndexSettings as IndexSettings from .settings import LanceDBSettings as LanceDBSettings from .settings import LLMSettings as LLMSettings from .settings import MemorySettings as MemorySettings +from .settings import MilvusSettings as MilvusSettings from .settings import MultimodalSettings as MultimodalSettings from .settings import RerankSettings as RerankSettings from .settings import Settings as Settings @@ -28,9 +31,11 @@ __all__ = [ "BoundaryDetectionSettings", "EmbeddingSettings", + "IndexSettings", "LLMSettings", "LanceDBSettings", "MemorySettings", + "MilvusSettings", "MultimodalSettings", "RerankSettings", "Settings", diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index e2137f84a..d67f0bc99 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -50,6 +50,24 @@ cache_size_kb = 2048 # Uncomment to override: # read_consistency_seconds = 5.0 +[index] +# Rebuildable vector/BM25 index backend. Markdown remains the source of truth +# and SQLite remains the system state store. +# Override via EVEROS_INDEX__BACKEND. +backend = "lancedb" + +[milvus] +# Optional derived index backend. Leave uri empty to use Milvus Lite at +# /.index/milvus/milvus.db. Set uri/token/db_name for Milvus server or +# Zilliz Cloud. +# Override via EVEROS_MILVUS__URI, EVEROS_MILVUS__TOKEN, EVEROS_MILVUS__DB_NAME. +uri = "" +token = "" +db_name = "" +consistency_level = "Session" +dimension = 1024 +collection_prefix = "everos" + [llm] # Provider-agnostic OpenAI-protocol client config. Override via env: # EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 50da9947d..5e36b390d 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -375,6 +375,30 @@ class CascadeSettings(BaseModel): optimize_rebuild_interval_seconds: float = 12 * 60 * 60.0 +class IndexSettings(BaseModel): + """Derived-index backend selection.""" + + backend: Literal["lancedb", "milvus"] = "lancedb" + + +class MilvusSettings(BaseModel): + """Remote Milvus Server or Zilliz Cloud connection settings. + + Embedded Milvus Lite is intentionally unsupported. Deployments that select + the Milvus backend must configure an external endpoint and should use a + unique database or collection prefix when sharing a cluster. + """ + + uri: str = "" + token: SecretStr = SecretStr("") + db_name: str = "" + consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = ( + "Session" + ) + dimension: int = Field(default=1024, ge=1) + collection_prefix: str = Field(default="everos", min_length=1) + + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" @@ -448,6 +472,8 @@ class Settings(BaseSettings): api: ApiSettings = ApiSettings() sqlite: SqliteSettings = SqliteSettings() lancedb: LanceDBSettings = LanceDBSettings() + index: IndexSettings = IndexSettings() + milvus: MilvusSettings = MilvusSettings() llm: LLMSettings = LLMSettings() embedding: EmbeddingSettings = EmbeddingSettings() rerank: RerankSettings = RerankSettings() diff --git a/src/everos/core/persistence/memory_root.py b/src/everos/core/persistence/memory_root.py index 701ff0012..5ca928c4f 100644 --- a/src/everos/core/persistence/memory_root.py +++ b/src/everos/core/persistence/memory_root.py @@ -11,6 +11,7 @@ .index/ derived indexes (rebuildable from markdown) sqlite/ system.db (+ WAL/SHM), ome.db, ome.aps.db lancedb/ LanceDB tables + milvus/ Milvus Lite database when Milvus is selected .tmp/ atomic-write staging directory .lock single-process lock anchor (created on demand by ``memory_root_lock``) @@ -165,6 +166,16 @@ def lancedb_dir(self) -> Path: """``/.index/lancedb/`` — LanceDB table root.""" return self.index_dir / "lancedb" + @property + def milvus_dir(self) -> Path: + """``/.index/milvus/`` — Milvus Lite derived index root.""" + return self.index_dir / "milvus" + + @property + def milvus_db(self) -> Path: + """``/.index/milvus/milvus.db`` — default Milvus Lite URI.""" + return self.milvus_dir / "milvus.db" + @property def sqlite_dir(self) -> Path: """``/.index/sqlite/`` — SQLite system DB root. @@ -244,10 +255,12 @@ def ensure(self) -> None: /.index/ /.index/sqlite/ /.index/lancedb/ + /.index/milvus/ /.tmp/ """ self.root.mkdir(parents=True, exist_ok=True) self.index_dir.mkdir(parents=True, exist_ok=True) self.sqlite_dir.mkdir(parents=True, exist_ok=True) self.lancedb_dir.mkdir(parents=True, exist_ok=True) + self.milvus_dir.mkdir(parents=True, exist_ok=True) self.tmp_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py new file mode 100644 index 000000000..486900058 --- /dev/null +++ b/src/everos/infra/persistence/index/__init__.py @@ -0,0 +1,329 @@ +"""Backend-neutral derived index persistence facade. + +Markdown is the source of truth and SQLite stores system state. This package +selects the rebuildable vector/BM25 index backend used by cascade, search, and +get. The default backend remains LanceDB; Milvus is opt-in through +``[index] backend = "milvus"``. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery + +try: + from lancedb.query import Occur +except ImportError: # pragma: no cover + from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] + +from everos.config import load_settings +from everos.infra.persistence import lancedb as _lancedb +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + ParentType, + UserProfile, +) + + +def active_backend() -> str: + """Return the configured derived index backend name.""" + return load_settings().index.backend + + +async def startup() -> Any: + """Initialise the configured derived index backend.""" + if active_backend() == "milvus": + milvus = _milvus() + await milvus.get_client() + await milvus.ensure_business_indexes() + return "milvus" + conn = await _lancedb.get_connection() + await _lancedb.verify_business_schemas() + await _lancedb.ensure_business_indexes() + return conn + + +async def shutdown() -> None: + """Dispose the configured derived index backend.""" + if active_backend() == "milvus": + await _milvus().dispose_connection() + else: + await _lancedb.dispose_connection() + + +async def ensure_business_indexes() -> None: + if active_backend() == "milvus": + await _milvus().ensure_business_indexes() + else: + await _lancedb.ensure_business_indexes() + + +async def verify_business_schemas() -> None: + if active_backend() == "milvus": + await _milvus().verify_business_schemas() + else: + await _lancedb.verify_business_schemas() + + +class _LanceIndexRepoAdapter: + """Add backend-neutral recall helpers to an existing LanceDB repo.""" + + def __init__(self, repo: Any, schema: type[Any]) -> None: + self._repo = repo + self.schema = schema + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo, name) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + fields = list(columns or self.schema.BM25_FIELDS) + if not query_terms or not fields: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + expr = _lance_expr(where) + best: dict[str, dict[str, Any]] = {} + for field in fields: + query = _build_or_query(query_terms, field) + rows = ( + await table.query() + .nearest_to_text(query) + .where(expr) + .limit(limit) + .to_list() + ) + for row in rows: + rid = row.get("id") + if not isinstance(rid, str): + continue + score = float(row.get("_score", 0.0)) + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + shaped = dict(row) + shaped["_score"] = score + best[rid] = shaped + return sorted( + best.values(), key=lambda row: float(row.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + ) -> list[dict[str, Any]]: + if not vector: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + return ( + await table.query() + .nearest_to(list(vector)) + .distance_type("cosine") + .where(_lance_expr(where)) + .limit(limit) + .to_list() + ) + + +class _IndexRepoRouter: + """Route repo calls to the configured derived index backend.""" + + def __init__( + self, lance_repo: Any, milvus_repo_name: str, schema: type[Any] + ) -> None: + self._lance = _LanceIndexRepoAdapter(lance_repo, schema) + self._milvus_repo_name = milvus_repo_name + self.schema = schema + + @property + def table_name(self) -> str: + return self.schema.TABLE_NAME + + def _repo(self) -> Any: + if active_backend() == "milvus": + return getattr(_milvus(), self._milvus_repo_name) + return self._lance + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo(), name) + + async def add(self, records: Sequence[Any]) -> None: + await self._repo().add(records) + + async def upsert(self, records: Sequence[Any], *, by: str = "id") -> None: + await self._repo().upsert(records, by=by) + + async def count(self) -> int: + return await self._repo().count() + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> Any: + return await self._repo().get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Any, *, limit: int = 100) -> list[Any]: + return await self._repo().find_where(_where_for_backend(where), limit=limit) + + async def find_one_where(self, where: Any) -> Any: + return await self._repo().find_one_where(_where_for_backend(where)) + + async def find_where_paginated( + self, + where: Any, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[Any], int]: + return await self._repo().find_where_paginated( + _where_for_backend(where), + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Any = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + return await self._repo().search( + vector=vector, where=_where_for_backend(where), limit=limit + ) + + async def update(self, updates: dict[str, Any], *, where: Any) -> None: + await self._repo().update(updates, where=_where_for_backend(where)) + + async def delete(self, predicate: Any) -> None: + await self._repo().delete(_where_for_backend(predicate)) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo().delete_by_md_path(md_path) + + async def optimize(self, **kwargs: Any) -> None: + await self._repo().optimize(**kwargs) + + async def rebuild_indexes(self) -> None: + await self._repo().rebuild_indexes() + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + return await self._repo().sparse_search( + query_terms, + _where_for_backend(where), + columns=columns, + limit=limit, + ) + + async def dense_search( + self, vector: Sequence[float], where: Any, *, limit: int + ) -> list[dict[str, Any]]: + return await self._repo().dense_search( + vector, _where_for_backend(where), limit=limit + ) + + +def _where_for_backend(where: Any) -> Any: + if active_backend() == "milvus": + return getattr(where, "milvus", where) + return _lance_expr(where) + + +def _lance_expr(where: Any) -> str: + return getattr(where, "lancedb", where) + + +def _build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery: + clean = [token for token in tokens if token] + if not clean: + return MatchQuery("", column=column) + if len(clean) == 1: + return MatchQuery(clean[0], column=column) + return BooleanQuery( + [(Occur.SHOULD, MatchQuery(token, column=column)) for token in clean] + ) + + +def _milvus() -> Any: + from everos.infra.persistence import milvus + + return milvus + + +episode_repo = _IndexRepoRouter(_lancedb.episode_repo, "episode_repo", Episode) +atomic_fact_repo = _IndexRepoRouter( + _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact +) +foresight_repo = _IndexRepoRouter( + _lancedb.foresight_repo, "foresight_repo", Foresight +) +agent_case_repo = _IndexRepoRouter( + _lancedb.agent_case_repo, "agent_case_repo", AgentCase +) +agent_skill_repo = _IndexRepoRouter( + _lancedb.agent_skill_repo, "agent_skill_repo", AgentSkill +) +user_profile_repo = _IndexRepoRouter( + _lancedb.user_profile_repo, "user_profile_repo", UserProfile +) +knowledge_topic_repo = _IndexRepoRouter( + _lancedb.knowledge_topic_repo, "knowledge_topic_repo", KnowledgeTopic +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "AgentCase", + "AgentSkill", + "AtomicFact", + "Episode", + "Foresight", + "KnowledgeTopic", + "ParentType", + "UserProfile", + "active_backend", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "shutdown", + "startup", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py new file mode 100644 index 000000000..4396360c8 --- /dev/null +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -0,0 +1,49 @@ +"""Milvus derived index backend. + +This package mirrors the LanceDB business index surface but stores rows in +Milvus collections. It is selected through ``Settings.index.backend`` and is +normally reached through :mod:`everos.infra.persistence.index`. +""" + +from __future__ import annotations + +from .milvus_manager import MilvusSchemaMismatchError as MilvusSchemaMismatchError +from .milvus_manager import dispose_connection as dispose_connection +from .milvus_manager import get_client as get_client +from .repos import ALL_REPOS as ALL_REPOS +from .repos import agent_case_repo as agent_case_repo +from .repos import agent_skill_repo as agent_skill_repo +from .repos import atomic_fact_repo as atomic_fact_repo +from .repos import episode_repo as episode_repo +from .repos import foresight_repo as foresight_repo +from .repos import knowledge_topic_repo as knowledge_topic_repo +from .repos import user_profile_repo as user_profile_repo + + +async def ensure_business_indexes() -> None: + """Create or verify every EverOS Milvus collection.""" + for repo in ALL_REPOS: + await repo.ensure_collection() + + +async def verify_business_schemas() -> None: + """Fail loud if existing Milvus collections drift from EverOS schemas.""" + for repo in ALL_REPOS: + await repo.verify_collection() + + +__all__ = [ + "ALL_REPOS", + "MilvusSchemaMismatchError", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "dispose_connection", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "get_client", + "knowledge_topic_repo", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/milvus/milvus_manager.py b/src/everos/infra/persistence/milvus/milvus_manager.py new file mode 100644 index 000000000..602e4a314 --- /dev/null +++ b/src/everos/infra/persistence/milvus/milvus_manager.py @@ -0,0 +1,87 @@ +"""Milvus connection and collection management for the derived index.""" + +from __future__ import annotations + +import re + +from pymilvus import MilvusClient + +from everos.config import MilvusSettings, load_settings +from everos.core.observability.logging import get_logger +from everos.core.persistence import MemoryRoot + +logger = get_logger(__name__) + +_client: MilvusClient | None = None + + +class MilvusSchemaMismatchError(RuntimeError): + """Raised when an existing Milvus collection does not match EverOS.""" + + +def collection_name(table_name: str, settings: MilvusSettings | None = None) -> str: + """Return the configured Milvus collection name for an EverOS table.""" + cfg = settings or load_settings().milvus + prefix = _sanitize_name_part(cfg.collection_prefix) + base = _sanitize_name_part(table_name) + name = f"{prefix}_{base}" if prefix else base + if not re.match(r"^[A-Za-z_]", name): + name = f"_{name}" + return name + + +async def get_client() -> MilvusClient: + """Return the process-wide MilvusClient, creating it lazily.""" + global _client + if _client is None: + settings = load_settings().milvus + uri = _resolve_uri(settings) + token = _secret(settings.token) + db_name = settings.db_name or "" + _client = MilvusClient(uri=uri, token=token, db_name=db_name) + logger.info( + "milvus_connection_opened", + uri=uri, + db_name=db_name or None, + consistency_level=settings.consistency_level, + ) + return _client + + +async def dispose_connection() -> None: + """Close the process-wide Milvus client.""" + global _client + if _client is not None: + _client.close() + _client = None + logger.info("milvus_connection_closed") + + +def _resolve_uri(settings: MilvusSettings) -> str: + if settings.uri: + return settings.uri + memory_root = MemoryRoot.default() + memory_root.milvus_dir.mkdir(parents=True, exist_ok=True) + return str(memory_root.milvus_db) + + +def _secret(value: object | None) -> str: + if value is None: + return "" + getter = getattr(value, "get_secret_value", None) + if callable(getter): + return getter() or "" + return str(value) + + +def _sanitize_name_part(value: str) -> str: + clean = re.sub(r"\W+", "_", value.strip()) + return clean.strip("_") + + +__all__ = [ + "MilvusSchemaMismatchError", + "collection_name", + "dispose_connection", + "get_client", +] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py new file mode 100644 index 000000000..826bc9267 --- /dev/null +++ b/src/everos/infra/persistence/milvus/repos.py @@ -0,0 +1,119 @@ +"""Milvus repo singletons for EverOS derived index tables.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + UserProfile, +) + +from .repository import MilvusRepoBase + + +class _EpisodeRepo(MilvusRepoBase[Episode]): + schema = Episode + + +class _AtomicFactRepo(MilvusRepoBase[AtomicFact]): + schema = AtomicFact + + +class _ForesightRepo(MilvusRepoBase[Foresight]): + schema = Foresight + + +class _AgentCaseRepo(MilvusRepoBase[AgentCase]): + schema = AgentCase + + +class _AgentSkillRepo(MilvusRepoBase[AgentSkill]): + schema = AgentSkill + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + rows = await self.find_where( + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=10_000, + ) + return len(rows) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[AgentSkill]: + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[AgentSkill]: + if not query_vector: + raise ValueError( + "query_vector must be non-empty; " + "call find_in_cluster for the scalar fallback" + ) + rows = await self.dense_search( + query_vector, + f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + limit=top_k, + ) + out: list[AgentSkill] = [] + for row in rows: + rid = row.get("id") + if isinstance(rid, str) and (item := await self.get_by_id(rid)) is not None: + out.append(item) + return out + + +class _UserProfileRepo(MilvusRepoBase[UserProfile]): + schema = UserProfile + + +class _KnowledgeTopicRepo(MilvusRepoBase[KnowledgeTopic]): + schema = KnowledgeTopic + + +def _q(value: str) -> str: + return value.replace("\\", "\\\\").replace("'", "\\'") + + +episode_repo = _EpisodeRepo() +atomic_fact_repo = _AtomicFactRepo() +foresight_repo = _ForesightRepo() +agent_case_repo = _AgentCaseRepo() +agent_skill_repo = _AgentSkillRepo() +user_profile_repo = _UserProfileRepo() +knowledge_topic_repo = _KnowledgeTopicRepo() + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py new file mode 100644 index 000000000..b8828118e --- /dev/null +++ b/src/everos/infra/persistence/milvus/repository.py @@ -0,0 +1,750 @@ +"""Milvus repository implementation for EverOS derived index tables.""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import re +from collections.abc import Sequence +from importlib.metadata import PackageNotFoundError, version +from typing import Any, ClassVar, get_args, get_origin + +from pymilvus import DataType, Function, FunctionType, MilvusClient + +from everos.component.utils.datetime import ( + ensure_utc, + from_iso_format, + from_timestamp, + to_timestamp_ms, +) +from everos.config import load_settings +from everos.core.observability.logging import get_logger +from everos.core.persistence import BaseLanceTable + +from .milvus_manager import ( + MilvusSchemaMismatchError, + collection_name, + get_client, +) + +logger = get_logger(__name__) + +_DUMMY_VECTOR_FIELD = "_everos_dummy_vector" +_SPARSE_SUFFIX = "__sparse" +_MAX_VARCHAR_LENGTH = 65_535 +_ID_MAX_LENGTH = 512 +_ARRAY_MAX_CAPACITY = 256 + + +def _q(value: str) -> str: + """Escape a string for a Milvus single-quoted expression literal.""" + return value.replace("\\", "\\\\").replace("'", "\\'") + + +class MilvusRepoBase[T: BaseLanceTable]: + """Generic Milvus repository for one EverOS derived index collection.""" + + schema: type[T] + _write_locks: ClassVar[dict[str, asyncio.Lock]] = {} + + @property + def table_name(self) -> str: + return self.schema.TABLE_NAME + + @property + def collection_name(self) -> str: + return collection_name(self.table_name) + + @classmethod + def _write_lock(cls, name: str) -> asyncio.Lock: + return cls._write_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_locks_for_tests(cls) -> None: + cls._write_locks.clear() + + async def ensure_collection(self) -> None: + client = await get_client() + if await _run(client.has_collection, self.collection_name): + await self.verify_collection() + return + + schema = self._build_collection_schema() + index_params = client.prepare_index_params() + if "vector" in self.schema.model_fields: + index_params.add_index( + field_name="vector", + index_type="AUTOINDEX", + metric_type="COSINE", + ) + else: + index_params.add_index( + field_name=_DUMMY_VECTOR_FIELD, + index_type="AUTOINDEX", + metric_type="COSINE", + ) + for field in self.schema.BM25_FIELDS: + index_params.add_index( + field_name=_sparse_field(field), + index_type="AUTOINDEX", + metric_type="BM25", + ) + + settings = load_settings().milvus + await _run( + client.create_collection, + collection_name=self.collection_name, + schema=schema, + index_params=index_params, + consistency_level=settings.consistency_level, + ) + logger.info( + "milvus_collection_created", + table=self.table_name, + collection=self.collection_name, + ) + + async def verify_collection(self) -> None: + client = await get_client() + description = await _run(client.describe_collection, self.collection_name) + actual = {field["name"] for field in description.get("fields", [])} + expected = set(self._stored_field_names()) + missing = expected - actual + if missing: + raise MilvusSchemaMismatchError( + f"Milvus collection {self.collection_name!r} schema drift: " + f"missing={sorted(missing)}. The index is rebuildable from md; " + "drop the collection and restart to rebuild it." + ) + + # ── Create / update ───────────────────────────────────────────── + + async def add(self, records: Sequence[T]) -> None: + if not records: + return + await self.ensure_collection() + client = await get_client() + payload = [self._to_milvus_record(record) for record in records] + async with self._write_lock(self.collection_name): + await _run(client.insert, self.collection_name, payload) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + if by != "id": + raise ValueError("MilvusRepoBase only supports upsert by id") + if not records: + return + await self.ensure_collection() + client = await get_client() + payload = [self._to_milvus_record(record) for record in records] + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, payload) + + async def update(self, updates: dict[str, Any], *, where: Any) -> None: + await self.ensure_collection() + rows = await self._query_raw(where, limit=10_000, include_vectors=True) + if not rows: + return + patched = [] + for row in rows: + merged = dict(row) + for key, value in updates.items(): + self._write_field_value(merged, key, value) + patched.append(merged) + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, patched) + + # ── Maintenance ──────────────────────────────────────────────── + + async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: + """Milvus indexes are managed by the service; no per-write compaction.""" + + async def rebuild_indexes(self) -> None: + """Milvus AUTOINDEX maintenance is service-managed.""" + + # ── Reads ────────────────────────────────────────────────────── + + async def count(self) -> int: + await self.ensure_collection() + client = await get_client() + stats = await _run(client.get_collection_stats, self.collection_name) + row_count = stats.get("row_count", 0) + return int(row_count) + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + if id_field != "id": + rows = await self.find_where(f"{id_field} = '{_q(id_value)}'", limit=1) + return rows[0] if rows else None + await self.ensure_collection() + client = await get_client() + rows = await _run( + client.get, + self.collection_name, + ids=[id_value], + output_fields=self._output_fields(include_vectors=True), + ) + if not rows: + return None + return self._model_from_milvus(rows[0]) + + async def find_where(self, where: Any, *, limit: int = 100) -> list[T]: + rows = await self._query_raw(where, limit=limit, include_vectors=True) + return [self._model_from_milvus(row) for row in rows] + + async def find_one_where(self, where: Any) -> T | None: + rows = await self.find_where(where, limit=1) + return rows[0] if rows else None + + async def find_where_paginated( + self, + where: Any, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + raw = await self._query_raw(where, limit=max_fetch, include_vectors=True) + total = len(raw) + if total >= max_fetch: + logger.warning( + "milvus_find_where_paginated_truncated", + table=self.table_name, + total=total, + max_fetch=max_fetch, + ) + rows = [self._model_from_milvus(row) for row in raw] + rows.sort( + key=lambda row: _sort_value(getattr(row, sort_by, None)), + reverse=descending, + ) + offset = (page - 1) * page_size + return rows[offset : offset + page_size], total + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(f"owner_id = '{_q(owner_id)}'", limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(f"md_path = '{_q(md_path)}'") + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Any = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + if vector is None: + return await self._query_candidate_rows(where, limit=limit) + return await self.dense_search(vector, where, limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + if not query_terms: + return [] + await self.ensure_collection() + fields = list(columns or self.schema.BM25_FIELDS) + if not fields: + return [] + client = await get_client() + expr = self._expr(where) + query = " ".join(term for term in query_terms if term) + best: dict[str, dict[str, Any]] = {} + for field in fields: + results = await _run( + client.search, + self.collection_name, + data=[query], + anns_field=_sparse_field(field), + filter=expr, + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "BM25"}, + ) + for row in _first_result_set(results): + shaped = self._candidate_row_from_search(row) + score = _bm25_score_from_distance(row.get("distance")) + shaped["_score"] = score + rid = shaped.get("id") + if not isinstance(rid, str): + continue + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + best[rid] = shaped + return sorted( + best.values(), key=lambda item: float(item.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + ) -> list[dict[str, Any]]: + if not vector or "vector" not in self.schema.model_fields: + return [] + await self.ensure_collection() + client = await get_client() + results = await _run( + client.search, + self.collection_name, + data=[list(vector)], + anns_field="vector", + filter=self._expr(where), + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "COSINE"}, + ) + return [ + self._candidate_row_from_search(row, normalize_cosine=True) + for row in _first_result_set(results) + ] + + # ── Deletes ──────────────────────────────────────────────────── + + async def delete(self, predicate: Any) -> None: + await self.ensure_collection() + client = await get_client() + async with self._write_lock(self.collection_name): + await _run( + client.delete, + self.collection_name, + filter=self._expr(predicate), + ) + + async def delete_by_md_path(self, md_path: str) -> int: + await self.ensure_collection() + before = await self.find_where(f"md_path = '{_q(md_path)}'", limit=10_000) + if not before: + return 0 + await self.delete(f"md_path = '{_q(md_path)}'") + return len(before) + + # ── Daily-log helpers ────────────────────────────────────────── + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + f"owner_id = '{_q(owner_id)}' AND entry_id = '{_q(entry_id)}' " + f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'" + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + quoted = ", ".join(f"'{_q(entry_id)}'" for entry_id in entry_ids) + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND entry_id IN ({quoted}) " + f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'", + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + f"owner_id = '{_q(owner_id)}' AND session_id = '{_q(session_id)}'", + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + f"parent_type = '{_q(parent_type)}' AND parent_id = '{_q(parent_id)}'", + limit=limit, + ) + + # ── Field conversion ─────────────────────────────────────────── + + def _build_collection_schema(self): # type: ignore[no-untyped-def] + settings = load_settings().milvus + schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) + has_vector = False + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + annotation = field.annotation + if name == "id": + schema.add_field( + field_name=name, + datatype=DataType.VARCHAR, + is_primary=True, + max_length=_ID_MAX_LENGTH, + ) + elif name == "vector": + has_vector = True + schema.add_field( + field_name=name, + datatype=DataType.FLOAT_VECTOR, + dim=settings.dimension, + ) + elif _is_datetime(annotation): + schema.add_field( + field_name=_datetime_storage_field(name), + datatype=DataType.INT64, + nullable=_is_optional(annotation), + ) + elif _is_list(annotation): + schema.add_field( + field_name=name, + datatype=DataType.ARRAY, + element_type=DataType.VARCHAR, + max_capacity=_ARRAY_MAX_CAPACITY, + max_length=_ID_MAX_LENGTH, + nullable=_is_optional(annotation), + ) + elif _is_float(annotation): + schema.add_field( + field_name=name, + datatype=DataType.DOUBLE, + nullable=_is_optional(annotation), + ) + elif _is_int(annotation): + schema.add_field( + field_name=name, + datatype=DataType.INT64, + nullable=_is_optional(annotation), + ) + else: + kwargs: dict[str, Any] = { + "field_name": name, + "datatype": DataType.VARCHAR, + "max_length": _MAX_VARCHAR_LENGTH, + } + if name in self.schema.BM25_FIELDS: + kwargs["enable_analyzer"] = True + elif _is_optional(annotation): + kwargs["nullable"] = True + schema.add_field(**kwargs) + + if not has_vector: + schema.add_field( + field_name=_DUMMY_VECTOR_FIELD, + datatype=DataType.FLOAT_VECTOR, + dim=1, + ) + for field in self.schema.BM25_FIELDS: + schema.add_field( + field_name=_sparse_field(field), + datatype=DataType.SPARSE_FLOAT_VECTOR, + ) + schema.add_function( + Function( + name=f"{field}_bm25", + function_type=FunctionType.BM25, + input_field_names=[field], + output_field_names=[_sparse_field(field)], + ) + ) + return schema + + def _stored_field_names(self) -> list[str]: + names: list[str] = [] + has_vector = False + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + if name == "vector": + has_vector = True + names.append(name) + elif _is_datetime(field.annotation): + names.append(_datetime_storage_field(name)) + else: + names.append(name) + if not has_vector: + names.append(_DUMMY_VECTOR_FIELD) + names.extend(_sparse_field(field) for field in self.schema.BM25_FIELDS) + return names + + def _output_fields(self, *, include_vectors: bool) -> list[str]: + fields: list[str] = [] + for name in self._stored_field_names(): + if name.endswith(_SPARSE_SUFFIX) or name == _DUMMY_VECTOR_FIELD: + continue + if not include_vectors and name == "vector": + continue + fields.append(name) + return fields + + def _to_milvus_record(self, record: T) -> dict[str, Any]: + raw = record.model_dump(mode="python") + out: dict[str, Any] = {} + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + value = raw.get(name) + if name in self.schema.BM25_FIELDS and value is None: + value = "" + if _is_datetime(field.annotation): + storage_name = _datetime_storage_field(name) + out[storage_name] = ( + _datetime_to_ms(value) if value is not None else None + ) + elif name == "vector": + out[name] = list(value or []) + elif _is_list(field.annotation): + out[name] = [str(item) for item in (value or [])] + else: + out[name] = value + if "vector" not in self.schema.model_fields: + out[_DUMMY_VECTOR_FIELD] = [0.0] + return out + + def _model_from_milvus(self, row: dict[str, Any]) -> T: + shaped = self._restore_row(row, include_distance=False) + return self.schema.model_validate(shaped) + + def _candidate_row_from_search( + self, row: dict[str, Any], *, normalize_cosine: bool = False + ) -> dict[str, Any]: + shaped = self._restore_row(row.get("entity", {}), include_distance=False) + raw_distance = row.get("distance") + shaped["_distance"] = ( + _cosine_distance_from_milvus(raw_distance) + if normalize_cosine + else raw_distance + ) + return shaped + + def _restore_row( + self, row: dict[str, Any], *, include_distance: bool + ) -> dict[str, Any]: + out: dict[str, Any] = {} + for name, field in self.schema.model_fields.items(): + if name == "subject_vector": + continue + if _is_datetime(field.annotation): + storage_name = _datetime_storage_field(name) + value = row.get(storage_name) + if value is None: + out[name] = None + else: + out[name] = from_timestamp(int(value)) + elif name in row: + value = row[name] + if name in self.schema.BM25_FIELDS and value == "" and _is_optional( + field.annotation + ): + out[name] = None + else: + out[name] = value + if include_distance and "distance" in row: + out["_distance"] = row["distance"] + return out + + def _write_field_value( + self, row: dict[str, Any], field_name: str, value: Any + ) -> None: + field = self.schema.model_fields.get(field_name) + if field is None: + row[field_name] = value + elif _is_datetime(field.annotation): + row[_datetime_storage_field(field_name)] = ( + _datetime_to_ms(value) if value is not None else None + ) + else: + row[field_name] = value + + async def _query_raw( + self, where: Any, *, limit: int, include_vectors: bool + ) -> list[dict[str, Any]]: + await self.ensure_collection() + client = await get_client() + return await _run( + client.query, + self.collection_name, + filter=self._expr(where), + output_fields=self._output_fields(include_vectors=include_vectors), + limit=limit, + ) + + async def _query_candidate_rows( + self, where: Any, *, limit: int + ) -> list[dict[str, Any]]: + rows = await self._query_raw(where, limit=limit, include_vectors=False) + return [self._restore_row(row, include_distance=False) for row in rows] + + def _expr(self, where: Any) -> str: + if where is None: + return "" + milvus_expr = getattr(where, "milvus", None) + if isinstance(milvus_expr, str): + return milvus_expr + if not isinstance(where, str): + return str(where) + return _lancedb_expr_to_milvus(where) + + +def _lancedb_expr_to_milvus(expr: str) -> str: + """Best-effort conversion for internal EverOS LanceDB predicates.""" + converted = expr + converted = re.sub(r"\barray_has\s*\(", "array_contains(", converted) + converted = re.sub(r"\bIS\s+NULL\b", "is null", converted, flags=re.IGNORECASE) + converted = re.sub( + r"\b([A-Za-z_][A-Za-z0-9_]*)\s+IN\s+\(([^)]*)\)", + lambda m: f"{m.group(1)} in [{m.group(2)}]", + converted, + flags=re.IGNORECASE, + ) + converted = re.sub(r"(?=])=(?!=)", "==", converted) + converted = re.sub(r"\btimestamp\b", "timestamp_ms", converted) + converted = re.sub(r"\bcreated_at\b", "created_at_ms", converted) + converted = re.sub(r"\bupdated_at\b", "updated_at_ms", converted) + converted = re.sub( + r"TIMESTAMP\s+'([^']*)'", + lambda m: str(_iso_to_ms(m.group(1))), + converted, + ) + return converted + + +def _datetime_storage_field(name: str) -> str: + return f"{name}_ms" + + +def _sparse_field(name: str) -> str: + return f"{name}{_SPARSE_SUFFIX}" + + +def _datetime_to_ms(value: Any) -> int: + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") + + +def _iso_to_ms(value: str) -> int: + aware = ensure_utc(from_iso_format(value)) + assert aware is not None + return to_timestamp_ms(aware) + + +def _is_optional(annotation: Any) -> bool: + args = get_args(annotation) + return type(None) in args + + +def _non_none_args(annotation: Any) -> tuple[Any, ...]: + return tuple(arg for arg in get_args(annotation) if arg is not type(None)) + + +def _is_datetime(annotation: Any) -> bool: + if annotation is dt.datetime: + return True + return any(arg is dt.datetime for arg in _non_none_args(annotation)) + + +def _is_list(annotation: Any) -> bool: + origin = get_origin(annotation) + if origin is list: + return True + return any(get_origin(arg) is list for arg in _non_none_args(annotation)) + + +def _is_float(annotation: Any) -> bool: + if annotation is float: + return True + return any(arg is float for arg in _non_none_args(annotation)) + + +def _is_int(annotation: Any) -> bool: + if annotation is int: + return True + return any(arg is int for arg in _non_none_args(annotation)) + + +def _sort_value(value: Any) -> Any: + if value is None: + fallback = ensure_utc(dt.datetime.min) + assert fallback is not None + return fallback + return value + + +def _bm25_score_from_distance(distance: Any) -> float: + """Convert the Milvus BM25 search distance into a higher-is-better score.""" + if distance is None: + return 0.0 + return abs(float(distance)) + + +def _cosine_distance_from_milvus(distance: Any) -> float | None: + """Normalize Milvus COSINE results to LanceDB-style distance. + + Milvus server / Zilliz Cloud return cosine similarity through the + ``distance`` field. Milvus Lite 3.0 reports cosine distance instead; keep + this version-gated workaround narrow because the upstream issue is expected + to be fixed after 3.0: + https://github.com/milvus-io/milvus-lite/issues/343 + + Recaller code expects LanceDB's ``1 - similarity`` distance contract. + """ + if distance is None: + return None + value = float(distance) + if not _uses_milvus_lite_3_0_cosine_distance_bug(): + value = 1.0 - value + if value < 0.0: + return 0.0 + if value > 1.0: + return 1.0 + return value + + +def _uses_milvus_lite() -> bool: + uri = load_settings().milvus.uri + if not uri: + return True + return "://" not in uri + + +def _uses_milvus_lite_3_0_cosine_distance_bug() -> bool: + if not _uses_milvus_lite(): + return False + return _milvus_lite_version() in {"3.0", "3.0.0"} + + +def _milvus_lite_version() -> str | None: + try: + return version("milvus-lite") + except PackageNotFoundError: + return None + + +def _first_result_set(results: Any) -> list[dict[str, Any]]: + if not results: + return [] + first = results[0] + return list(first or []) + + +async def _run(func: Any, /, *args: Any, **kwargs: Any) -> Any: + return await asyncio.to_thread(func, *args, **kwargs) + + +__all__ = ["MilvusRepoBase"] diff --git a/src/everos/memory/cascade/handlers/agent_case.py b/src/everos/memory/cascade/handlers/agent_case.py index a6fa9c5b7..2f6bcca20 100644 --- a/src/everos/memory/cascade/handlers/agent_case.py +++ b/src/everos/memory/cascade/handlers/agent_case.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AgentCase, ParentType, agent_case_repo +from everos.infra.persistence.index import AgentCase, ParentType, agent_case_repo from ._common import require_float, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 88ae83959..25315d35b 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -42,7 +42,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import AgentSkill, agent_skill_repo from everos.infra.persistence.markdown import AgentSkillFrontmatter from ..types import HandlerOutcome diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index 6dea35e4e..673423369 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -23,7 +23,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AtomicFact, ParentType, atomic_fact_repo +from everos.infra.persistence.index import AtomicFact, ParentType, atomic_fact_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index 84878d302..57dd6e5ad 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -38,7 +38,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Episode, ParentType, episode_repo +from everos.infra.persistence.index import Episode, ParentType, episode_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index adbefe020..02aeea926 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Foresight, ParentType, foresight_repo +from everos.infra.persistence.index import Foresight, ParentType, foresight_repo from ._common import ( optional_int, diff --git a/src/everos/memory/cascade/handlers/knowledge_topic.py b/src/everos/memory/cascade/handlers/knowledge_topic.py index daa8fbc3c..2535a0572 100644 --- a/src/everos/memory/cascade/handlers/knowledge_topic.py +++ b/src/everos/memory/cascade/handlers/knowledge_topic.py @@ -38,7 +38,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, ParsedMarkdown -from everos.infra.persistence.lancedb import KnowledgeTopic, knowledge_topic_repo +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from everos.infra.persistence.sqlite import ( TopicUpsertPayload, knowledge_topic_sqlite_repo, diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 4842986d0..8d7573521 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -27,7 +27,7 @@ from typing import Any, ClassVar from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import UserProfile, user_profile_repo +from everos.infra.persistence.index import UserProfile, user_profile_repo from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 diff --git a/src/everos/memory/cascade/registry.py b/src/everos/memory/cascade/registry.py index 8537575e6..3054a17a3 100644 --- a/src/everos/memory/cascade/registry.py +++ b/src/everos/memory/cascade/registry.py @@ -18,7 +18,7 @@ from pathlib import PurePosixPath from everos.core.persistence.markdown import BaseFrontmatter -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 407a06bc7..46b1e6305 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -16,6 +16,7 @@ from __future__ import annotations from everos.memory.search import FilterNode, compile_filters +from everos.memory.search.filters import BackendFilters, compile_filters_for_backends def compile_filters_for_get( @@ -38,3 +39,21 @@ def compile_filters_for_get( app_id=app_id, project_id=project_id, ) + + +def compile_filters_for_get_backends( + filters: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> BackendFilters: + """Compile ``/get`` filters for every derived index backend.""" + return compile_filters_for_backends( + filters, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index c42edd7ea..7a9421797 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -12,7 +12,7 @@ Reads only — never writes. Filters are compiled through :func:`compile_filters_for_get` so the column allow-list stays shared with :mod:`memory.search`. Pagination + in-memory sort -runs through :meth:`LanceRepoBase.find_where_paginated`. +runs through the configured derived index repository. """ from __future__ import annotations @@ -34,11 +34,11 @@ GetRequest, GetResponse, ) -from .filters_adapter import compile_filters_for_get +from .filters_adapter import compile_filters_for_get_backends if TYPE_CHECKING: from everos.core.persistence.lancedb import LanceRepoBase - from everos.infra.persistence.lancedb import ( + from everos.infra.persistence.index import ( AgentCase, AgentSkill, Episode, @@ -49,7 +49,7 @@ class GetManager: - """Dispatch ``GetRequest`` to the matching LanceDB-backed repo and + """Dispatch ``GetRequest`` to the matching derived index repo and shape rows into the public DTO.""" def __init__( @@ -70,7 +70,7 @@ def __init__( async def get(self, req: GetRequest) -> GetResponse: request_id = resolve_request_id() descending = req.sort_order == "desc" - where = compile_filters_for_get( + where = compile_filters_for_get_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, @@ -187,7 +187,7 @@ def _shape_agent_skill(row: AgentSkill) -> GetAgentSkillItem: async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: """Fetch the owner's single profile row from the ``user_profile`` - LanceDB table (kept in sync with ``users//user.md`` by cascade). + Derived index row (kept in sync with ``users//user.md`` by cascade). Profile is one-row-per-owner KV — there is no pagination / sort / filter surface, so at most one item is returned. Mirrors the diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 2dd9aa804..22df21a47 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,4 +1,4 @@ -"""Filters DSL → LanceDB ``where`` string compiler. +"""Filters DSL → derived-index filter compiler. The Filters DSL is intentionally permissive at the JSON layer (so callers can pass whatever they like and get a clean 400 if it is not supported) @@ -23,7 +23,8 @@ kind). Iterate / membership-test only; do not mutate. * :data:`RESERVED_FIELDS` — names rejected inside any ``filters`` block. * :func:`compile_predicate` — render one ``{field: value}`` clause to - SQL. Operator-map and equality-shorthand are both handled. + a backend-specific predicate. Operator-map and equality-shorthand are + both handled. The high-level :func:`compile_filters` remains the entry point for ``/search`` (combinator-aware). @@ -31,10 +32,18 @@ from __future__ import annotations +import dataclasses import datetime as _dt +import json from typing import Any, Final -from everos.component.utils.datetime import from_timestamp, to_iso_format +from everos.component.utils.datetime import ( + ensure_utc, + from_iso_format, + from_timestamp, + to_iso_format, + to_timestamp_ms, +) from everos.core.errors import FilterError as FilterError from .dto import FilterNode @@ -50,6 +59,15 @@ "lte": "<=", "in": "IN", } +_MILVUS_OP_MAP: Final[dict[str, str]] = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", + "in": "in", +} # Field kinds: ``str`` rendered as ``''``; ``ts`` rendered as # ``TIMESTAMP ''`` (DataFusion timestamp literal); ``array_str`` @@ -82,6 +100,17 @@ def __init__(self, column: str, kind: _FieldKind) -> None: ) +@dataclasses.dataclass(frozen=True) +class BackendFilters: + """Compiled filters for every supported derived index backend.""" + + lancedb: str + milvus: str + + def __str__(self) -> str: + return self.lancedb + + # ── Public API ─────────────────────────────────────────────────────────── @@ -102,19 +131,72 @@ def compile_filters( from another — omitting it would let a query bleed across spaces. Both ``/search`` and ``/get`` share this compile path. """ - base = [ - f"owner_id = '{_escape_str(owner_id)}'", - f"owner_type = '{owner_type}'", - f"app_id = '{_escape_str(app_id)}'", - f"project_id = '{_escape_str(project_id)}'", - ] + return _compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="lancedb", + ) + + +def compile_filters_for_backends( + node: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> BackendFilters: + """Compile request filters for every supported derived index backend.""" + return BackendFilters( + lancedb=_compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="lancedb", + ), + milvus=_compile_filters_backend( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend="milvus", + ), + ) + + +def _compile_filters_backend( + node: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + backend: str, +) -> str: + base = _base_clauses( + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + backend=backend, + ) # Only episode / atomic_fact tables carry the ``deprecated_by`` column # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": - base.append("deprecated_by IS NULL") + base.append( + "deprecated_by IS NULL" + if backend == "lancedb" + else "deprecated_by is null" + ) if node is None: return " AND ".join(base) - compiled = _compile_node(node.model_dump(exclude_none=True)) + compiled = _compile_node(node.model_dump(exclude_none=True), backend=backend) if not compiled: return " AND ".join(base) return " AND ".join([*base, compiled]) @@ -123,7 +205,7 @@ def compile_filters( # ── Internals ──────────────────────────────────────────────────────────── -def _compile_node(raw: dict[str, Any]) -> str: +def _compile_node(raw: dict[str, Any], *, backend: str = "lancedb") -> str: """Walk one DSL node; return the matching SQL fragment (no leading parens). Empty nodes yield ``""`` so :func:`compile_filters` can skip the @@ -133,9 +215,9 @@ def _compile_node(raw: dict[str, Any]) -> str: parts: list[str] = [] if (and_list := raw.pop("AND", None)) is not None: - parts.append(_compile_combinator(and_list, "AND")) + parts.append(_compile_combinator(and_list, "AND", backend=backend)) if (or_list := raw.pop("OR", None)) is not None: - parts.append(_compile_combinator(or_list, "OR")) + parts.append(_compile_combinator(or_list, "OR", backend=backend)) for field, value in raw.items(): if field in RESERVED_FIELDS: @@ -144,7 +226,7 @@ def _compile_node(raw: dict[str, Any]) -> str: ) if field not in ALLOWED_FIELDS: raise FilterError(f"unsupported filter field: {field!r}") - parts.append(compile_predicate(field, value)) + parts.append(compile_predicate(field, value, backend=backend)) # Drop empty fragments coming from empty AND/OR arrays. parts = [p for p in parts if p] @@ -155,7 +237,9 @@ def _compile_node(raw: dict[str, Any]) -> str: return " AND ".join(parts) -def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: +def _compile_combinator( + children: list[dict[str, Any]], op: str, *, backend: str = "lancedb" +) -> str: """Render an ``AND`` / ``OR`` array of child nodes.""" if not isinstance(children, list): raise FilterError(f"{op} expects an array of nodes") @@ -163,7 +247,7 @@ def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: for child in children: if not isinstance(child, dict): raise FilterError(f"{op} children must be objects") - compiled = _compile_node(child) + compiled = _compile_node(child, backend=backend) if compiled: fragments.append(f"({compiled})") if not fragments: @@ -175,7 +259,7 @@ def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: return "(" + glue.join(fragments) + ")" -def compile_predicate(field: str, value: Any) -> str: +def compile_predicate(field: str, value: Any, *, backend: str = "lancedb") -> str: """Render one ``"": `` clause to SQL. Public primitive — :mod:`memory.get` builds a flat (no AND/OR) @@ -195,48 +279,65 @@ def compile_predicate(field: str, value: Any) -> str: if not value: raise FilterError(f"empty operator map for field {field!r}") clauses = [ - _compile_op_clause(spec, field, op, op_val) for op, op_val in value.items() + _compile_op_clause(spec, field, op, op_val, backend=backend) + for op, op_val in value.items() ] if len(clauses) == 1: return clauses[0] return "(" + " AND ".join(clauses) + ")" # Equality shorthand. - return _compile_op_clause(spec, field, "eq", value) + return _compile_op_clause(spec, field, "eq", value, backend=backend) -def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> str: +def _compile_op_clause( + spec: _FieldSpec, field: str, op: str, value: Any, *, backend: str = "lancedb" +) -> str: """Render a single `` `` clause.""" if op not in _OP_MAP: raise FilterError(f"unsupported operator {op!r} on field {field!r}") - sql_op = _OP_MAP[op] + op_map = _MILVUS_OP_MAP if backend == "milvus" else _OP_MAP + sql_op = op_map[op] + column = _column_for_backend(spec, backend) if spec.kind == "array_str": # Only equality / membership make sense on a list column. + fn = "array_contains" if backend == "milvus" else "array_has" if op == "eq": - literal = _escape_str(_require_str(value, field)) - return f"array_has({spec.column}, '{literal}')" + literal = _render_str_literal(_require_str(value, field), backend) + return f"{fn}({column}, {literal})" if op == "in": items = _require_list(value, field) - literals = [f"'{_escape_str(_require_str(v, field))}'" for v in items] - inner = " OR ".join(f"array_has({spec.column}, {lit})" for lit in literals) + literals = [ + _render_str_literal(_require_str(v, field), backend) for v in items + ] + inner = " OR ".join(f"{fn}({column}, {lit})" for lit in literals) return f"({inner})" raise FilterError(f"operator {op!r} is not supported on array field {field!r}") if op == "in": items = _require_list(value, field) - literals = [_render_literal(v, spec.kind, field) for v in items] - return f"{spec.column} IN ({', '.join(literals)})" + literals = [ + _render_literal(v, spec.kind, field, backend=backend) for v in items + ] + if backend == "milvus": + return f"{column} in [{', '.join(literals)}]" + return f"{column} IN ({', '.join(literals)})" - return f"{spec.column} {sql_op} {_render_literal(value, spec.kind, field)}" + literal = _render_literal(value, spec.kind, field, backend=backend) + return f"{column} {sql_op} {literal}" # ── Literal rendering ──────────────────────────────────────────────────── -def _render_literal(value: Any, kind: _FieldKind, field: str) -> str: +def _render_literal( + value: Any, kind: _FieldKind, field: str, *, backend: str = "lancedb" +) -> str: if kind == "str": - return f"'{_escape_str(_require_str(value, field))}'" + return _render_str_literal(_require_str(value, field), backend) if kind == "ts": + if backend == "milvus": + return str(_render_ts_ms(value, field)) return f"TIMESTAMP '{_render_ts(value, field)}'" raise FilterError(f"unsupported field kind {kind!r} for field {field!r}") @@ -257,6 +358,58 @@ def _render_ts(value: Any, field: str) -> str: raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") +def _render_ts_ms(value: Any, field: str) -> int: + if isinstance(value, bool): + raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") + if isinstance(value, (int, float)): + return int(value) + if isinstance(value, str): + if "'" in value: + raise FilterError(f"timestamp string for {field!r} contains a quote") + try: + aware = ensure_utc(from_iso_format(value)) + except (TypeError, ValueError) as exc: + raise FilterError( + f"timestamp value for {field!r} must be ms or ISO string" + ) from exc + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, _dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") + + +def _render_str_literal(value: str, backend: str) -> str: + if backend == "milvus": + return json.dumps(value, ensure_ascii=False) + return f"'{_escape_str(value)}'" + + +def _base_clauses( + *, + owner_id: str, + owner_type: str, + app_id: str, + project_id: str, + backend: str, +) -> list[str]: + eq = "==" if backend == "milvus" else "=" + return [ + f"owner_id {eq} {_render_str_literal(owner_id, backend)}", + f"owner_type {eq} {_render_str_literal(owner_type, backend)}", + f"app_id {eq} {_render_str_literal(app_id, backend)}", + f"project_id {eq} {_render_str_literal(project_id, backend)}", + ] + + +def _column_for_backend(spec: _FieldSpec, backend: str) -> str: + if backend == "milvus" and spec.kind == "ts": + return f"{spec.column}_ms" + return spec.column + + def _escape_str(value: str) -> str: """Double single quotes — SQL-standard escape for a single-quoted literal.""" return value.replace("'", "''") diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 1ae5bfd91..7bb15a6b4 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -22,7 +22,8 @@ no query-relevance score we can assign to a fact pulled by parent_id alone, and emitting ``score=0.0`` facts would muddy the contract. -The manager never writes to storage; it only reads LanceDB + markdown. +The manager never writes to storage; it only reads the derived index + +markdown. """ from __future__ import annotations @@ -70,7 +71,7 @@ SearchResponse, UnprocessedMessageDTO, ) -from .filters import compile_filters +from .filters import compile_filters_for_backends from .hierarchy import build_ep_to_fact_parents, heap_expand from .shaper import ( reshape_hybrid_output, @@ -204,7 +205,7 @@ async def search(self, req: SearchRequest) -> SearchResponse: # Compile filters first: a malformed `filters` payload is a user # input error (422) and should surface before the server-side # component guard (500). The two steps are independent. - where = compile_filters( + where = compile_filters_for_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, diff --git a/src/everos/memory/search/recall/agent_case.py b/src/everos/memory/search/recall/agent_case.py index 93a7f9b60..9c17ba45e 100644 --- a/src/everos/memory/search/recall/agent_case.py +++ b/src/everos/memory/search/recall/agent_case.py @@ -13,17 +13,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentCase, get_table +from everos.infra.persistence.index import AgentCase, agent_case_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -50,43 +48,15 @@ async def sparse_recall( per BM25 column (``MatchQuery`` is column-bound), then the two per-column result lists merge by id keeping the max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentCase.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentCase.BM25_FIELDS), + merged_rows = await agent_case_repo.sparse_search( + terms, + where, + columns=AgentCase.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep the max BM25 score across the two columns. - # task_intent hits typically score higher (the retrieval anchor); - # approach hits catch queries that match a step detail. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -97,15 +67,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_case_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index f444e4a38..0c1831c79 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -9,17 +9,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentSkill, get_table +from everos.infra.persistence.index import AgentSkill, agent_skill_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -48,41 +46,15 @@ async def sparse_recall( rationale. One BooleanQuery per BM25 column; merge by id with max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentSkill.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentSkill.BM25_FIELDS), + merged_rows = await agent_skill_repo.sparse_search( + terms, + where, + columns=AgentSkill.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep max BM25 score across the two columns. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -93,15 +65,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_skill_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -124,8 +88,7 @@ async def fetch_by_case_ids( """ if not case_ids: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) clause = " OR ".join(f"array_has(source_case_ids, '{_q(c)}')" for c in case_ids) full_where = f"({where}) AND ({clause})" - rows = await table.query().where(full_where).limit(limit).to_list() + rows = await agent_skill_repo.search(where=full_where, limit=limit) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 69eba22d4..38ec3872a 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -21,11 +21,10 @@ from everalgo.types import Candidate, FactCandidate -from everos.infra.persistence.lancedb import AtomicFact, get_table +from everos.infra.persistence.index import AtomicFact, atomic_fact_repo from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -49,14 +48,14 @@ async def sparse_recall( self, query: str, where: str, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery (see EpisodeRecaller docstring).""" - bq = build_or_query( - self._deps.tokenizer, query, column=AtomicFact.BM25_FIELDS[0] - ) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await atomic_fact_repo.sparse_search( + terms, + where, + columns=AtomicFact.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -78,15 +77,7 @@ async def dense_recall( """ if not vector: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await atomic_fact_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -170,17 +161,11 @@ async def _query_facts_for_parents( clause = f"parent_id IN ({quoted})" full_where = f"({where}) AND ({clause})" limit = per_episode * max(len(parent_to_eps), 1) - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) if query_vector: - return await ( - table.query() - .nearest_to(list(query_vector)) - .distance_type("cosine") - .where(full_where) - .limit(limit) - .to_list() + return await atomic_fact_repo.dense_search( + query_vector, full_where, limit=limit ) - return await table.query().where(full_where).limit(limit).to_list() + return await atomic_fact_repo.search(where=full_where, limit=limit) def _build_parent_to_episode_map( diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index a2766dcec..1f2c13821 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,11 +7,10 @@ from everalgo.types import Candidate -from everos.infra.persistence.lancedb import Episode, get_table +from everos.infra.persistence.index import Episode, active_backend, episode_repo from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -55,12 +54,14 @@ async def sparse_recall( Mirrors enterprise's ``bool.should + minimum_should_match=1`` ES design. """ - bq = build_or_query(self._deps.tokenizer, query, column=Episode.BM25_FIELDS[0]) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await episode_repo.sparse_search( + terms, + where, + columns=Episode.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -72,16 +73,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await episode_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -113,6 +105,10 @@ async def dense_recall_subject( """ if not vector: return [] + if active_backend() == "milvus": + return await self.dense_recall(vector, where, limit=limit) + from everos.infra.persistence.lancedb import get_table + table = await get_table(Episode.TABLE_NAME, Episode) rows = ( await table.query() @@ -149,8 +145,7 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: No ``limit`` — the full owner partition is required for cluster membership matching. """ - table = await get_table(Episode.TABLE_NAME, Episode) - rows = await table.query().where(where).to_list() + rows = await episode_repo.search(where=where, limit=10_000) result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") @@ -173,8 +168,7 @@ async def fetch_by_entry_ids( """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" if not entry_ids: return [] - table = await get_table(Episode.TABLE_NAME, Episode) quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) full_where = f"({where}) AND (entry_id IN ({quoted}))" - rows = await table.query().where(full_where).limit(len(entry_ids)).to_list() + rows = await episode_repo.search(where=full_where, limit=len(entry_ids)) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py index ea484e17a..9cc6a9bfb 100644 --- a/src/everos/memory/search/recall/knowledge_topic.py +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -12,17 +12,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import KnowledgeTopic, get_table +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -78,26 +76,15 @@ async def sparse_recall( matching the query in either its summary or its content body is surfaced without double-counting. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, KnowledgeTopic.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in KnowledgeTopic.BM25_FIELDS), + merged_rows = await knowledge_topic_repo.sparse_search( + terms, + where, + columns=KnowledgeTopic.BM25_FIELDS, + limit=limit, ) - merged_rows = _merge_bm25_results(per_column, limit=limit) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -109,15 +96,7 @@ async def dense_recall( """Cosine ANN over the ``summary`` vector (1024-d).""" if not vector: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await knowledge_topic_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/profile.py b/src/everos/memory/search/recall/profile.py index be149b927..640d27091 100644 --- a/src/everos/memory/search/recall/profile.py +++ b/src/everos/memory/search/recall/profile.py @@ -19,7 +19,7 @@ from typing import Any from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import user_profile_repo +from everos.infra.persistence.index import user_profile_repo from ..dto import SearchProfileItem diff --git a/src/everos/memory/strategies/extract_agent_skill.py b/src/everos/memory/strategies/extract_agent_skill.py index 7d4e6bc3b..cb840ea3e 100644 --- a/src/everos/memory/strategies/extract_agent_skill.py +++ b/src/everos/memory/strategies/extract_agent_skill.py @@ -83,10 +83,10 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase as LanceAgentCase, ) -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, ) diff --git a/src/everos/memory/strategies/extract_user_profile.py b/src/everos/memory/strategies/extract_user_profile.py index 711eca096..38e7db5d4 100644 --- a/src/everos/memory/strategies/extract_user_profile.py +++ b/src/everos/memory/strategies/extract_user_profile.py @@ -59,7 +59,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import BaseEvent from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import episode_repo +from everos.infra.persistence.index import episode_repo from everos.infra.persistence.markdown import ( ProfileReader, ProfileWriter, diff --git a/src/everos/memory/strategies/reflect_episodes.py b/src/everos/memory/strategies/reflect_episodes.py index 2d39b8795..44aa0e149 100644 --- a/src/everos/memory/strategies/reflect_episodes.py +++ b/src/everos/memory/strategies/reflect_episodes.py @@ -22,7 +22,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import CronTick from everos.infra.ome.triggers import Cron -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( atomic_fact_repo, episode_repo, ) diff --git a/src/everos/service/get.py b/src/everos/service/get.py index 6d5a72bc2..0a83ff746 100644 --- a/src/everos/service/get.py +++ b/src/everos/service/get.py @@ -1,8 +1,8 @@ """Get use case — lazy singleton wiring for ``POST /api/v2/memory/get``. Mirrors :mod:`everos.service.search`: the :class:`GetManager` and its -LanceDB repo singletons are built on first call so the FastAPI module -import order stays decoupled from the lifespan that brings up LanceDB. +derived index repo singletons are built on first call so the FastAPI module +import order stays decoupled from the lifespan that brings up the index backend. ``/get`` is read-only and uses no embedding / LLM / rerank clients — it never blocks on optional components the way ``/search`` does. @@ -11,7 +11,7 @@ from __future__ import annotations from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, episode_repo, diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index fb4fbb17f..b02d2e78b 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -120,6 +120,33 @@ def test_embedding_rerank_defaults() -> None: assert s.llm.api_key.get_secret_value() == "" +def test_index_milvus_defaults() -> None: + s = Settings() + assert s.index.backend == "lancedb" + assert s.milvus.uri == "" + assert s.milvus.token.get_secret_value() == "" + assert s.milvus.db_name == "" + assert s.milvus.consistency_level == "Session" + assert s.milvus.dimension == 1024 + assert s.milvus.collection_prefix == "everos" + + +def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://localhost:19530") + monkeypatch.setenv("EVEROS_MILVUS__TOKEN", "secret") + monkeypatch.setenv("EVEROS_MILVUS__DB_NAME", "tenant_a") + monkeypatch.setenv("EVEROS_MILVUS__CONSISTENCY_LEVEL", "Strong") + monkeypatch.setenv("EVEROS_MILVUS__DIMENSION", "1536") + s = Settings() + assert s.index.backend == "milvus" + assert s.milvus.uri == "http://localhost:19530" + assert s.milvus.token.get_secret_value() == "secret" + assert s.milvus.db_name == "tenant_a" + assert s.milvus.consistency_level == "Strong" + assert s.milvus.dimension == 1536 + + def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: """No --root, no EVEROS_ROOT → ~/.everos.""" monkeypatch.delenv("EVEROS_ROOT", raising=False) diff --git a/tests/unit/test_core/test_persistence/test_memory_root.py b/tests/unit/test_core/test_persistence/test_memory_root.py index 6da6ccd9a..efca98227 100644 --- a/tests/unit/test_core/test_persistence/test_memory_root.py +++ b/tests/unit/test_core/test_persistence/test_memory_root.py @@ -91,6 +91,8 @@ def test_dotfile_paths(tmp_path: Path) -> None: mr = MemoryRoot(tmp_path) assert mr.index_dir == tmp_path / ".index" assert mr.lancedb_dir == tmp_path / ".index" / "lancedb" + assert mr.milvus_dir == tmp_path / ".index" / "milvus" + assert mr.milvus_db == tmp_path / ".index" / "milvus" / "milvus.db" assert mr.sqlite_dir == tmp_path / ".index" / "sqlite" assert mr.system_db == tmp_path / ".index" / "sqlite" / "system.db" assert mr.lock_file == tmp_path / ".lock" @@ -104,6 +106,7 @@ def test_ensure_creates_required_dirs(tmp_path: Path) -> None: assert mr.index_dir.is_dir() assert mr.sqlite_dir.is_dir() assert mr.lancedb_dir.is_dir() + assert mr.milvus_dir.is_dir() assert mr.tmp_dir.is_dir() # User-visible dirs are NOT pre-created. assert not mr.agents_dir().exists() diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py new file mode 100644 index 000000000..901d67034 --- /dev/null +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -0,0 +1,127 @@ +"""Real Milvus Lite coverage for the optional derived index backend.""" + +from __future__ import annotations + +import datetime as dt +import importlib.util +import uuid +from pathlib import Path + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("pymilvus") is None, + reason="pymilvus optional extra is not installed", +) + + +def test_milvus_score_normalization_handles_lite_and_cloud( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from everos.infra.persistence.milvus import repository + + monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(-1.5) == pytest.approx(1.5) + + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.0") + assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) + + monkeypatch.setenv("EVEROS_MILVUS__URI", "https://example.zillizcloud.com") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) + + monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) + monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.1") + load_settings.cache_clear() + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + + +@pytest_asyncio.fixture(autouse=True) +async def _milvus_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", f"test_{uuid.uuid4().hex}") + load_settings.cache_clear() + yield + from everos.infra.persistence.index import shutdown + + await shutdown() + load_settings.cache_clear() + + +async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: + from everos.infra.persistence.index import Episode, episode_repo + from everos.memory.search.filters import compile_filters_for_backends + + v_apple = [0.0] * 1024 + v_apple[0] = 1.0 + v_banana = [0.0] * 1024 + v_banana[1] = 1.0 + now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + + await episode_repo.upsert( + [ + Episode( + id="u1_ep1", + entry_id="ep1", + owner_id="u1", + owner_type="user", + app_id="default", + project_id="default", + session_id="s1", + timestamp=now, + parent_id="mc1", + sender_ids=["user"], + episode="red apple memory", + episode_tokens="red apple memory", + md_path="default_app/default_project/users/u1/episodes/day.md", + content_sha256="a", + vector=v_apple, + ), + Episode( + id="u1_ep2", + entry_id="ep2", + owner_id="u1", + owner_type="user", + app_id="default", + project_id="default", + session_id="s1", + timestamp=now + dt.timedelta(seconds=1), + parent_id="mc2", + sender_ids=["assistant"], + episode="blue banana memory", + episode_tokens="blue banana memory", + md_path="default_app/default_project/users/u1/episodes/day.md", + content_sha256="b", + vector=v_banana, + ), + ] + ) + + filters = compile_filters_for_backends(None, owner_id="u1", owner_type="user") + rows = await episode_repo.find_where(filters, limit=10) + assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} + assert rows[0].timestamp.utcoffset() == dt.timedelta(0) + + sparse = await episode_repo.sparse_search( + ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 + ) + assert [row["id"] for row in sparse] == ["u1_ep1"] + assert sparse[0]["_score"] > 0 + + dense = await episode_repo.dense_search(v_apple, filters, limit=5) + assert dense[0]["id"] == "u1_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0) + + deleted = await episode_repo.delete_by_md_path( + "default_app/default_project/users/u1/episodes/day.md" + ) + assert deleted == 2 + assert await episode_repo.count() == 0 diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 35cec6b79..4cc5e9bfb 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -56,7 +56,7 @@ class _StubRepo: async def find_where_paginated( self, - where: str, + where: Any, *, sort_by: str, descending: bool = True, @@ -65,7 +65,7 @@ async def find_where_paginated( max_fetch: int = 20000, ) -> tuple[list[Any], int]: self.last = _CallRecord( - where=where, + where=str(where), sort_by=sort_by, descending=descending, page=page, diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 808c2a382..a4267d8c8 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -9,6 +9,7 @@ FilterNode, compile_filters, ) +from everos.memory.search.filters import compile_filters_for_backends # ── Base injection ─────────────────────────────────────────────────────── @@ -266,3 +267,35 @@ def test_compile_filters_excludes_deprecated_by_for_user() -> None: def test_compile_filters_omits_deprecated_by_for_agent() -> None: result = compile_filters(None, owner_id="agent_1", owner_type="agent") assert "deprecated_by" not in result + + +# ── Backend-specific rendering ───────────────────────────────────────── + + +def test_compile_filters_for_backends_preserves_lancedb_default() -> None: + filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") + assert str(filters) == filters.lancedb + assert "owner_id = 'u_a'" in filters.lancedb + assert "owner_id == \"u_a\"" in filters.milvus + assert "deprecated_by IS NULL" in filters.lancedb + assert "deprecated_by is null" in filters.milvus + + +def test_compile_filters_for_milvus_timestamp_and_array() -> None: + node = FilterNode.model_validate( + {"timestamp": {"gte": 1704067200000}, "sender_id": "u_jason"} + ) + filters = compile_filters_for_backends(node, owner_id="u_a", owner_type="user") + assert "timestamp_ms >= 1704067200000" in filters.milvus + assert 'array_contains(sender_ids, "u_jason")' in filters.milvus + assert "TIMESTAMP '" in filters.lancedb + assert "array_has(sender_ids, 'u_jason')" in filters.lancedb + + +def test_compile_filters_for_milvus_escapes_string_literals() -> None: + node = FilterNode.model_validate({"session_id": "ses's"}) + filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") + assert "owner_id == \"al'ice\"" in filters.milvus + assert "session_id == \"ses's\"" in filters.milvus + assert "owner_id = 'al''ice'" in filters.lancedb + assert "session_id = 'ses''s'" in filters.lancedb diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 73dd66611..9ee62af13 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -106,25 +106,25 @@ def __init__(self, sparse: list[Candidate], dense: list[Candidate]) -> None: self.last_where: str | None = None async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = str(where) return list(self._sparse[:limit]) async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = str(where) return list(self._dense[:limit]) async def fetch_by_parent_ids( - self, parent_ids: Sequence[str], where: str + self, parent_ids: Sequence[str], where: Any ) -> list[Candidate]: by_parent = {str(c.metadata.get("parent_id", "")): c for c in self._dense} return [by_parent[p] for p in parent_ids if p in by_parent] async def fetch_by_entry_ids( - self, entry_ids: Sequence[str], where: str + self, entry_ids: Sequence[str], where: Any ) -> list[Candidate]: by_entry = {str(c.metadata.get("entry_id", "")): c for c in self._dense} return [by_entry[e] for e in entry_ids if e in by_entry] @@ -152,7 +152,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Any, *, per_episode: int, query_vector: Any = None, @@ -203,7 +203,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: return list(self._dense) async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Any, *, limit: int ) -> list[Candidate]: return list(self._by_case) diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 4549322d8..2cde9f368 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -58,9 +58,9 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( _make_row("ep_2", "mc_2"), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -75,9 +75,9 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( """metadata['episode_id'] carries the real LanceDB episode id for final shaping.""" rows = [_make_row("ep_abc", "mc_xyz")] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -104,9 +104,9 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( }, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -131,9 +131,9 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -156,9 +156,9 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): result = await recaller.fetch_all_for_owner("owner_id = 'alice'") @@ -179,14 +179,10 @@ async def test_fetch_by_entry_ids_returns_candidates( entry_id="entry_xyz", ), ] - mock_tbl = MagicMock() - mock_tbl.query.return_value.where.return_value.limit.return_value.to_list = ( - AsyncMock(return_value=rows) - ) with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=mock_tbl, + return_value=rows, ): result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") @@ -226,9 +222,9 @@ async def test_sparse_recall_as_child_injects_parent_id( {**_make_row("ep_1", "mc_1", entry_id="entry_1"), "_score": 1.0}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table(rows), + return_value=rows, ): result = await recaller.sparse_recall_as_child( "hello", "owner_id = 'alice'", limit=10 @@ -256,9 +252,9 @@ async def test_sparse_recall_as_child_falls_back_to_id_when_no_entry_id( "_score": 0.5, } with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table([row]), + return_value=[row], ): result = await recaller.sparse_recall_as_child( "hello", "owner_id = 'alice'", limit=10 @@ -286,9 +282,9 @@ async def test_dense_recall_as_child_injects_parent_id( {**_make_row("ep_3", "mc_3", entry_id="entry_3"), "_distance": 0.1}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_ann_table(rows), + return_value=rows, ): result = await recaller.dense_recall_as_child( [0.1] * 1024, "owner_id = 'alice'", limit=10 @@ -325,10 +321,16 @@ async def test_dense_recall_subject_returns_subject_vector_source( rows = [ {**_make_row("ep_s1", "mc_s1", entry_id="entry_s1"), "_distance": 0.2}, ] - with patch( - "everos.memory.search.recall.episode.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + with ( + patch( + "everos.memory.search.recall.episode.active_backend", + return_value="lancedb", + ), + patch( + "everos.infra.persistence.lancedb.get_table", + new_callable=AsyncMock, + return_value=_mock_subject_ann_table(rows), + ), ): result = await recaller.dense_recall_subject( [0.1] * 1024, "owner_id = 'alice'", limit=10 @@ -354,10 +356,16 @@ async def test_dense_recall_subject_as_child_injects_parent_id( rows = [ {**_make_row("ep_s2", "mc_s2", entry_id="entry_s2"), "_distance": 0.15}, ] - with patch( - "everos.memory.search.recall.episode.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + with ( + patch( + "everos.memory.search.recall.episode.active_backend", + return_value="lancedb", + ), + patch( + "everos.infra.persistence.lancedb.get_table", + new_callable=AsyncMock, + return_value=_mock_subject_ann_table(rows), + ), ): result = await recaller.dense_recall_subject_as_child( [0.1] * 1024, "owner_id = 'alice'", limit=10 diff --git a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py index d40d8a41e..3e98c58b4 100644 --- a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py +++ b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py @@ -1,11 +1,11 @@ """Unit tests for ``KnowledgeTopicRecaller``. -Verifies dual-column BM25 + cosine ANN recall, using ``unittest.mock`` -to patch ``get_table`` so no real LanceDB connection is needed. +Verifies dual-column BM25 delegation + cosine ANN recall, using +``unittest.mock`` to patch the backend-neutral index repo. White-box surfaces touched: - - ``everos.memory.search.recall.knowledge_topic.get_table`` (patched) - - ``KnowledgeTopicRecaller.sparse_recall`` — queries both BM25 columns + - ``everos.memory.search.recall.knowledge_topic.knowledge_topic_repo`` (patched) + - ``KnowledgeTopicRecaller.sparse_recall`` — passes both BM25 columns - ``KnowledgeTopicRecaller.dense_recall`` — cosine ANN with distance→score """ @@ -17,6 +17,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index import KnowledgeTopic from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.knowledge_topic import KnowledgeTopicRecaller @@ -58,41 +59,6 @@ def _make_row( return row -def _mock_bm25_table( - summary_rows: list[dict[str, Any]], - content_rows: list[dict[str, Any]], -) -> MagicMock: - """Build a table mock whose BM25 results differ per column. - - The first ``nearest_to_text`` call (summary_tokens) returns - ``summary_rows``; the second (content_tokens) returns ``content_rows``. - ``asyncio.gather`` fires both concurrently, so we use ``side_effect`` - on the chain rather than recording call order. - """ - summary_chain = MagicMock() - summary_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=summary_rows - ) - - content_chain = MagicMock() - content_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=content_rows - ) - - tbl = MagicMock() - tbl.query.return_value.nearest_to_text.side_effect = [summary_chain, content_chain] - return tbl - - -def _mock_ann_table(rows: list[dict[str, Any]]) -> MagicMock: - """Build a table mock for ANN (dense) queries.""" - tbl = MagicMock() - ann = tbl.query.return_value.nearest_to.return_value - chain = ann.distance_type.return_value.where.return_value.limit.return_value - chain.to_list = AsyncMock(return_value=rows) - return tbl - - @pytest.fixture() def recaller() -> KnowledgeTopicRecaller: return KnowledgeTopicRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) @@ -109,16 +75,20 @@ def recaller() -> KnowledgeTopicRecaller: async def test_sparse_recall_queries_both_columns( recaller: KnowledgeTopicRecaller, ) -> None: - """``nearest_to_text`` must be called once per BM25 column.""" - tbl = _mock_bm25_table( - summary_rows=[_make_row("t1", score=0.9)], - content_rows=[_make_row("t2", score=0.7)], - ) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + """Both BM25 columns must be delegated to the index repo.""" + rows = [_make_row("t1", score=0.9), _make_row("t2", score=0.7)] + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ) as mock_sparse: result = await recaller.sparse_recall("topic query", _WHERE, limit=10) - # nearest_to_text called twice (once per column) - assert tbl.query.return_value.nearest_to_text.call_count == 2 + mock_sparse.assert_awaited_once() + assert list(mock_sparse.await_args.args[0]) == ["topic", "query"] + assert mock_sparse.await_args.args[1] == _WHERE + assert mock_sparse.await_args.kwargs["columns"] == KnowledgeTopic.BM25_FIELDS + assert mock_sparse.await_args.kwargs["limit"] == 10 ids = {c.id for c in result} assert ids == {"t1", "t2"} @@ -126,13 +96,14 @@ async def test_sparse_recall_queries_both_columns( async def test_sparse_recall_merges_by_max_score( recaller: KnowledgeTopicRecaller, ) -> None: - """When the same id appears in both columns, keep the higher score.""" + """Scores returned by the repo are preserved on keyword candidates.""" shared_id = "topic_shared" - summary_rows = [_make_row(shared_id, score=0.5)] - content_rows = [_make_row(shared_id, score=0.9)] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=[_make_row(shared_id, score=0.9)], + ): result = await recaller.sparse_recall("overlap", _WHERE, limit=10) assert len(result) == 1 @@ -144,16 +115,16 @@ async def test_sparse_recall_merges_by_max_score( async def test_sparse_recall_returns_sorted_by_score( recaller: KnowledgeTopicRecaller, ) -> None: - """Merged results must be sorted descending by score, truncated to limit.""" - summary_rows = [ - _make_row("a", score=0.3), + """The recaller preserves repo ordering and maps rows to candidates.""" + rows = [ _make_row("b", score=0.8), - ] - content_rows = [ _make_row("c", score=0.6), ] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.sparse_recall("query", _WHERE, limit=2) assert len(result) == 2 @@ -164,16 +135,19 @@ async def test_sparse_recall_returns_sorted_by_score( async def test_sparse_recall_empty_query_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty tokenisation short-circuits — no LanceDB query is issued.""" + """Empty tokenisation short-circuits — no repo query is issued.""" tok = MagicMock(spec=Tokenizer) tok.tokenize.return_value = [] r = KnowledgeTopicRecaller(RecallerDeps(tokenizer=tok)) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + ) as mock_sparse: result = await r.sparse_recall("", _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_sparse.assert_not_called() # --------------------------------------------------------------------------- @@ -189,8 +163,11 @@ async def test_dense_recall_cosine_conversion( _make_row("t1", distance=0.2), _make_row("t2", distance=0.5), ] - tbl = _mock_ann_table(rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=10) assert len(result) == 2 @@ -203,12 +180,15 @@ async def test_dense_recall_cosine_conversion( async def test_dense_recall_empty_vector_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty vector short-circuits — no LanceDB query is issued.""" - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + """Empty vector short-circuits — no repo query is issued.""" + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + ) as mock_dense: result = await recaller.dense_recall([], _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_dense.assert_not_called() async def test_dense_recall_metadata_excludes_noise_columns( @@ -218,8 +198,11 @@ async def test_dense_recall_metadata_excludes_noise_columns( row = _make_row("t1", distance=0.3) row["vector"] = [0.0] * 1024 - tbl = _mock_ann_table([row]) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=[row], + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=5) assert len(result) == 1 diff --git a/uv.lock b/uv.lock index ec09b1abb..c9c12429c 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,14 @@ version = 1 revision = 3 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] [[package]] name = "aiosqlite" @@ -99,6 +107,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/55/af/861ebc2e318a5c3300e3eb63bc4d30f3d70a46d13b360093728ac0705eed/cachetools-7.1.6.tar.gz", hash = "sha256:c7a79e7f30ba9943c1cefd08cc36f006aaae086e017af9166f1d59d6170c47e1", size = 40572, upload-time = "2026-07-23T22:47:53.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/f2/2086ba18a925a73586c4d4e61d25f4a6058e56fd00d77ce8f1d361ab4c9b/cachetools-7.1.6-py3-none-any.whl", hash = "sha256:2c12e255780330af28b91bb7fb96cce4c766f04e38396b9a24510190a5827096", size = 16954, upload-time = "2026-07-23T22:47:52.397Z" }, +] + [[package]] name = "cairocffi" version = "1.7.1" @@ -595,6 +612,9 @@ dependencies = [ ] [package.optional-dependencies] +milvus = [ + { name = "pymilvus", extra = ["milvus-lite"] }, +] multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, ] @@ -641,6 +661,7 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, + { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=3.0.0" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -651,7 +672,7 @@ requires-dist = [ { name = "watchdog", specifier = ">=4.0.0" }, { name = "watchfiles", specifier = ">=0.21.0" }, ] -provides-extras = ["multimodal", "otel"] +provides-extras = ["multimodal", "otel", "milvus"] [package.metadata.requires-dev] dev = [ @@ -677,6 +698,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] +[[package]] +name = "faiss-cpu" +version = "1.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, + { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" }, +] + [[package]] name = "fastapi" version = "0.139.0" @@ -850,6 +888,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/8f/774ce522de6a7e70fbeceeaeb6fbe502f5dfb8365728fb3bb4cb23463da8/grimp-3.14-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a424ad14d5deb56721ac24ab939747f72ab3d378d42e7d1f038317d33b052b77", size = 2515157, upload-time = "2025-12-10T17:54:55.874Z" }, ] +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, + { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" }, + { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1286,6 +1365,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] +[[package]] +name = "milvus-lite" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "faiss-cpu" }, + { name = "grpcio" }, + { name = "numpy" }, + { name = "pyarrow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/58/54c72981958073d365febeae8fed3115c87761f45905f2d4725f506b94d1/milvus_lite-3.1.0.tar.gz", hash = "sha256:8d467ae81173f1ede93723a80f08d85b91988b451cd2b7bbf9b5a7cfb875e04b", size = 636180, upload-time = "2026-07-15T06:40:58.177Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/c2/fb5c2b73618973f943c8064e2d1f076f81731e2fafd5575a6ddaf3deda68/milvus_lite-3.1.0-py3-none-any.whl", hash = "sha256:0d20fffbee3c683eca69c4732a091273123368452159581ed050c933b41a59be", size = 252267, upload-time = "2026-07-15T06:40:52.538Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -1446,6 +1540,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "orjson" +version = "3.11.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, + { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, + { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, + { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, + { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, + { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, + { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, + { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, + { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, + { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, + { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, + { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, + { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, + { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, + { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, + { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, + { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, + { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, + { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, + { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, + { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, + { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, + { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, + { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, + { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -1455,6 +1602,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "parso" version = "0.8.7" @@ -1878,6 +2071,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/97/03635143a12a5d941f545548b00f8ac39d35565321a2effb4154ed267338/pyinstrument-5.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:b6a71f5e7f53c86c9b476b30cf19509463a63581ef17ddbd8680fee37ae509db", size = 128164, upload-time = "2026-01-04T18:38:32.281Z" }, ] +[[package]] +name = "pymilvus" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "grpcio" }, + { name = "orjson" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "setuptools" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/c1/01647e61f3a82fd881382746b6dde3401d65b88cd4f75bd059901fb2392b/pymilvus-3.0.0-1-py3-none-any.whl", hash = "sha256:57c8e7c87fbbf579f122b4df893949bc78e50bca2988527864891bd544817b05", size = 344817, upload-time = "2026-05-07T14:57:45.235Z" }, +] + +[package.optional-dependencies] +milvus-lite = [ + { name = "milvus-lite", marker = "sys_platform != 'win32'" }, +] + [[package]] name = "pytest" version = "9.0.3" @@ -2183,6 +2399,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + [[package]] name = "shellingham" version = "1.5.4" From c1d5baa9e016c0b05e0b61bc9ce7b4e9b0d6904d Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Thu, 9 Jul 2026 10:33:00 +0000 Subject: [PATCH 2/5] fix(milvus): bound fact recall and use exact counts --- .../infra/persistence/milvus/repository.py | 12 ++++++-- .../memory/search/recall/atomic_fact.py | 6 +++- .../unit/test_infra/test_milvus/test_repo.py | 1 + .../test_search/test_recall_atomic_fact.py | 29 +++++++++++++++++++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index b8828118e..90019f414 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -167,9 +167,15 @@ async def rebuild_indexes(self) -> None: async def count(self) -> int: await self.ensure_collection() client = await get_client() - stats = await _run(client.get_collection_stats, self.collection_name) - row_count = stats.get("row_count", 0) - return int(row_count) + rows = await _run( + client.query, + self.collection_name, + filter="", + output_fields=["count(*)"], + ) + if not rows: + return 0 + return int(rows[0].get("count(*)", 0)) async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: if id_field != "id": diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 38ec3872a..f78b3f50a 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -32,6 +32,7 @@ _NOISE_COLUMNS = frozenset( {"vector", "_distance", "_score", "created_at", "updated_at"} ) +_MAX_FACT_RECALL_LIMIT = 1024 class AtomicFactRecaller: @@ -160,7 +161,10 @@ async def _query_facts_for_parents( quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) clause = f"parent_id IN ({quoted})" full_where = f"({where}) AND ({clause})" - limit = per_episode * max(len(parent_to_eps), 1) + # Milvus server / Zilliz Cloud reject search topK values above 1024. + # The fact expansion only needs a bounded candidate pool for top-N + # competition, so keep the same cap for every derived-index backend. + limit = min(per_episode * max(len(parent_to_eps), 1), _MAX_FACT_RECALL_LIMIT) if query_vector: return await atomic_fact_repo.dense_search( query_vector, full_where, limit=limit diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 901d67034..8d265d81f 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -109,6 +109,7 @@ async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: rows = await episode_repo.find_where(filters, limit=10) assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} assert rows[0].timestamp.utcoffset() == dt.timedelta(0) + assert await episode_repo.count() == 2 sparse = await episode_repo.sparse_search( ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index f2e80b88e..c3c71d433 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -29,6 +29,7 @@ atomic_fact_repo, lancedb_manager, ) +from everos.memory.search.recall import atomic_fact as atomic_fact_module from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -267,6 +268,34 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: assert out["alice_ep_a"][0].score == 0.0 +async def test_facts_for_episodes_caps_dense_recall_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Large parent fan-out must not exceed Milvus/Zilliz search topK limits.""" + captured: dict[str, int] = {} + + async def fake_dense_search(vector, where, *, limit): # type: ignore[no-untyped-def] + captured["limit"] = limit + return [] + + monkeypatch.setattr( + atomic_fact_module.atomic_fact_repo, + "dense_search", + fake_dense_search, + ) + ep_to_parents = {f"alice_ep_{i}": [f"parent_{i}"] for i in range(80)} + + out = await _recaller().facts_for_episodes( + ep_to_parents, + "owner_id = 'alice' AND owner_type = 'user'", + per_episode=20, + query_vector=_unit_vector(0), + ) + + assert out == {} + assert captured["limit"] == 1024 + + # ── Dual parent_id (post-1.5 migration) ──────────────────────────────── From e6983075e20017f0bd1585e6ab16ee16ded321db Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Fri, 10 Jul 2026 03:39:14 +0000 Subject: [PATCH 3/5] style: format Milvus index changes --- src/everos/infra/persistence/index/__init__.py | 4 +--- src/everos/infra/persistence/milvus/repository.py | 6 ++++-- src/everos/memory/search/filters.py | 4 +--- tests/unit/test_memory/test_search/test_filters.py | 6 +++--- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index 486900058..0cc969eb6 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -278,9 +278,7 @@ def _milvus() -> Any: atomic_fact_repo = _IndexRepoRouter( _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact ) -foresight_repo = _IndexRepoRouter( - _lancedb.foresight_repo, "foresight_repo", Foresight -) +foresight_repo = _IndexRepoRouter(_lancedb.foresight_repo, "foresight_repo", Foresight) agent_case_repo = _IndexRepoRouter( _lancedb.agent_case_repo, "agent_case_repo", AgentCase ) diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index 90019f414..de9143b3d 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -550,8 +550,10 @@ def _restore_row( out[name] = from_timestamp(int(value)) elif name in row: value = row[name] - if name in self.schema.BM25_FIELDS and value == "" and _is_optional( - field.annotation + if ( + name in self.schema.BM25_FIELDS + and value == "" + and _is_optional(field.annotation) ): out[name] = None else: diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 22df21a47..0f417afc0 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -190,9 +190,7 @@ def _compile_filters_backend( # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": base.append( - "deprecated_by IS NULL" - if backend == "lancedb" - else "deprecated_by is null" + "deprecated_by IS NULL" if backend == "lancedb" else "deprecated_by is null" ) if node is None: return " AND ".join(base) diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index a4267d8c8..59e85dd2e 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -276,7 +276,7 @@ def test_compile_filters_for_backends_preserves_lancedb_default() -> None: filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") assert str(filters) == filters.lancedb assert "owner_id = 'u_a'" in filters.lancedb - assert "owner_id == \"u_a\"" in filters.milvus + assert 'owner_id == "u_a"' in filters.milvus assert "deprecated_by IS NULL" in filters.lancedb assert "deprecated_by is null" in filters.milvus @@ -295,7 +295,7 @@ def test_compile_filters_for_milvus_timestamp_and_array() -> None: def test_compile_filters_for_milvus_escapes_string_literals() -> None: node = FilterNode.model_validate({"session_id": "ses's"}) filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") - assert "owner_id == \"al'ice\"" in filters.milvus - assert "session_id == \"ses's\"" in filters.milvus + assert 'owner_id == "al\'ice"' in filters.milvus + assert 'session_id == "ses\'s"' in filters.milvus assert "owner_id = 'al''ice'" in filters.lancedb assert "session_id = 'ses''s'" in filters.lancedb From 0d383394931a2dcd3f651dda3e7d8b3ad736d996 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Tue, 18 Aug 2026 12:34:57 +0000 Subject: [PATCH 4/5] refactor(index): require remote Milvus deployments --- .github/workflows/ci.yml | 43 ++ config.example.toml | 12 +- docs/cascade_runbook.md | 13 +- docs/configuration.md | 23 +- docs/storage_layout.md | 10 +- pyproject.toml | 5 + src/everos/config/default.toml | 8 +- src/everos/config/settings.py | 16 +- src/everos/core/persistence/memory_root.py | 13 - .../entrypoints/api/lifespans/lancedb.py | 29 +- .../entrypoints/cli/commands/cascade.py | 47 +- .../infra/persistence/index/__init__.py | 82 ++- .../infra/persistence/index/predicate.py | 45 ++ src/everos/infra/persistence/index/schema.py | 341 +++++++++ .../infra/persistence/lancedb/predicate.py | 83 +++ .../infra/persistence/milvus/__init__.py | 36 + .../persistence/milvus/milvus_manager.py | 30 +- .../infra/persistence/milvus/predicate.py | 100 +++ src/everos/infra/persistence/milvus/repos.py | 61 +- .../infra/persistence/milvus/repository.py | 668 +++++++++--------- src/everos/infra/persistence/predicate.py | 138 ++++ src/everos/memory/cascade/_backfill.py | 8 +- .../cascade/handlers/_daily_log_base.py | 24 +- .../memory/cascade/handlers/agent_skill.py | 15 +- src/everos/memory/get/filters_adapter.py | 7 +- src/everos/memory/reflection/orchestrator.py | 63 +- src/everos/memory/search/filters.py | 438 ++++-------- .../memory/search/recall/agent_skill.py | 25 +- .../memory/search/recall/atomic_fact.py | 24 +- src/everos/memory/search/recall/episode.py | 61 +- src/everos/service/knowledge.py | 11 +- .../test_cascade_cli_integration.py | 2 +- tests/integration/test_milvus_remote.py | 217 ++++++ .../test_reflection_integration.py | 17 + .../test_tiers/test_upgrade_path.py | 18 +- tests/unit/test_config/test_settings.py | 3 - .../test_persistence/test_memory_root.py | 3 - .../unit/test_infra/test_milvus/test_repo.py | 232 +++--- .../test_backfill_subject_null_recovery.py | 12 +- .../test_cascade/test_handler_agent_case.py | 15 +- .../test_cascade/test_handler_agent_skill.py | 30 +- .../test_cascade/test_handler_episode.py | 13 +- .../test_get/test_filters_adapter.py | 26 +- .../unit/test_memory/test_get/test_manager.py | 3 +- .../test_memory/test_search/test_filters.py | 67 +- .../test_memory/test_search/test_manager.py | 5 +- .../test_search/test_recall_agent_skill.py | 3 +- .../test_search/test_recall_atomic_fact.py | 29 +- .../test_search/test_recall_episode.py | 78 +- .../test_search/test_recall_or_semantics.py | 7 +- .../test_service/test_knowledge_search.py | 11 +- uv.lock | 161 ++--- 52 files changed, 2204 insertions(+), 1227 deletions(-) create mode 100644 src/everos/infra/persistence/index/predicate.py create mode 100644 src/everos/infra/persistence/index/schema.py create mode 100644 src/everos/infra/persistence/lancedb/predicate.py create mode 100644 src/everos/infra/persistence/milvus/predicate.py create mode 100644 src/everos/infra/persistence/predicate.py create mode 100644 tests/integration/test_milvus_remote.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b90d55c9f..7d1fdd3b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,49 @@ jobs: - name: Integration tests run: make integration + milvus-integration: + name: Milvus Server integration + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies (frozen) + run: make install-deps + + - name: Start Milvus Server + run: | + curl -fsSL \ + https://github.com/milvus-io/milvus/releases/download/v2.6.22/milvus-standalone-docker-compose.yml \ + -o /tmp/milvus-compose.yml + docker compose -f /tmp/milvus-compose.yml up -d + for attempt in $(seq 1 90); do + if curl -fsS http://127.0.0.1:9091/healthz; then + exit 0 + fi + sleep 2 + done + docker compose -f /tmp/milvus-compose.yml logs + exit 1 + + - name: Milvus derived-index contract + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + EVEROS_TEST_MILVUS_FULL_STARTUP: "1" + run: uv run pytest tests/integration/test_milvus_remote.py -v + + - name: Stop Milvus Server + if: always() + run: docker compose -f /tmp/milvus-compose.yml down -v + package: name: package build runs-on: ubuntu-latest diff --git a/config.example.toml b/config.example.toml index 0145dd686..00045b2a9 100644 --- a/config.example.toml +++ b/config.example.toml @@ -59,16 +59,16 @@ max_concurrent = 5 # ── Optional Milvus derived index ───────────────────── # Markdown remains the source of truth. Set the index backend to Milvus -# when you want the rebuildable vector/BM25 index to live in Milvus. -# Leave uri empty for Milvus Lite at /.index/milvus/milvus.db. +# when you want the rebuildable vector/BM25 index to live in an external +# Milvus Server or Zilliz Cloud deployment. Embedded Milvus Lite and local +# database paths are not supported. # # [index] # backend = "milvus" # # [milvus] -# uri = "" # or "http://localhost:19530" +# uri = "http://localhost:19530" # required remote endpoint # token = "" # required for Zilliz Cloud/auth-enabled Milvus -# db_name = "" +# db_name = "" # use a dedicated database when available # consistency_level = "Session" -# dimension = 1024 # must match your embedding model output -# collection_prefix = "everos" +# collection_prefix = "everos" # make unique on a shared database diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index d836672f9..338d2a7cf 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -127,7 +127,7 @@ parallel with a live `everos server`. ## Rebuild the index: `everos cascade rebuild` -The safe recovery from a drifted or corrupt LanceDB index. It rebuilds +The safe recovery from a drifted or corrupt derived index. It rebuilds the whole index from markdown (the source of truth) in one shot: ```bash @@ -136,15 +136,16 @@ everos cascade rebuild --yes # non-interactive ``` > **Stop the `everos server` first.** Unlike `cascade sync`, rebuild -> **drops and recreates** the LanceDB tables. A running daemon holds +> **drops and recreates** the active backend's tables or collections. A running +> daemon holds > cached table handles that would keep pointing at (and writing to) the > dropped dataset, corrupting the rebuild. This is the one cascade > command that is **not** safe to run alongside a live server. What it does, in order: -1. **Drops** every business LanceDB table (`drop_business_tables`) and - evicts them from the connection cache. +1. **Drops** every business table or collection (`drop_business_tables`) and + evicts it from the process cache. 2. **Recreates** them empty from the current schema + FTS indexes (`ensure_business_indexes`). 3. **Clears** the cascade queue (`md_change_state.reset_all`) so every @@ -164,6 +165,10 @@ Why not a bare `rm`: | `rm -rf .index` | ✅ | ❌ deletes un-extracted messages | | `everos cascade rebuild` | ✅ | ✅ | +For a remote Milvus backend, rebuild acts only on collections whose names use +the configured `collection_prefix`. Use a unique prefix or dedicated database +before running it on a shared Milvus Server or Zilliz Cloud deployment. + ## Recovery paths ### LanceDB schema drift on startup diff --git a/docs/configuration.md b/docs/configuration.md index 6848d7839..0608170e9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,16 +108,27 @@ everos init --root /data/everos ### `[milvus]` Milvus is optional. Install with `everos[milvus]`, then set -`[index].backend = "milvus"`. +`[index].backend = "milvus"`. The backend connects only to an external +Milvus Server or Zilliz Cloud endpoint; embedded Milvus Lite and local `.db` +paths are not supported. | Field | Type | Default | Description | |---|---|---|---| -| `uri` | string \| null | `""` | Empty uses Milvus Lite at `/.index/milvus/milvus.db`. Set to a Milvus server or Zilliz Cloud endpoint to use an external service. | -| `token` | string \| null | `""` | Token for Zilliz Cloud or auth-enabled Milvus. | -| `db_name` | string \| null | `""` | Optional Milvus database name. | +| `uri` | string | `""` | Required when the Milvus backend is selected. Must be a remote Milvus Server or Zilliz Cloud endpoint. | +| `token` | string | `""` | Token for Zilliz Cloud or auth-enabled Milvus Server. | +| `db_name` | string | `""` | Optional Milvus database name. Prefer a dedicated database on shared clusters. | | `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | -| `dimension` | int | `1024` | Dense vector dimension. Must match the configured embedding model output. | -| `collection_prefix` | string | `"everos"` | Prefix for EverOS Milvus collections. | +| `collection_prefix` | string | `"everos"` | Prefix for EverOS collections. Must start with a letter or underscore and contain only letters, digits, and underscores. Make it unique when deployments share a database. | + +EverOS creates seven collections and supports every dense field in the shared +index schema, including `episode.subject_vector`. The vector dimension is owned +by that schema (currently 1024), rather than duplicated in Milvus config. + +Milvus imposes limits that LanceDB does not: strings are capped at 65,535 +UTF-8 bytes, string arrays at 256 items, and each array item at 512 UTF-8 +bytes. EverOS validates these before a write and reports the exact table and +field instead of relying on a server-side insert error. Ensure the selected +Zilliz Cloud plan permits at least seven collections. ### `[llm]` diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 71447905a..3d90c775d 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -52,8 +52,6 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ └── ome.db.lock OME single-engine guard (portalocker) │ ├── lancedb/ │ │ └── .lance/ default derived index backend -│ └── milvus/ -│ └── milvus.db Milvus Lite database when configured │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -67,7 +65,7 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). The path manager is [`MemoryRoot`](../src/everos/core/persistence/memory_root.py), exposing every path as a property. `MemoryRoot.ensure()` creates the -runtime-required dirs (`.index/{sqlite,lancedb,milvus}/`, `.tmp/`); the +runtime-required dirs (`.index/{sqlite,lancedb}/`, `.tmp/`); the user-visible dirs are *not* pre-created — they appear on first write. Config files (`everos.toml`, `ome.toml`) are created by `everos init`. @@ -183,8 +181,6 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ unprocessed_buffer, conversation_status, cluster) ├── lancedb/ │ └── .lance/ default derived index backend -└── milvus/ - └── milvus.db Milvus Lite database when configured ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -197,8 +193,8 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers - The **derived index backend** holds the per-kind business rows, keyed `_` (so cross-table joins use `(owner_id, entry_id)`). LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled - as the same rebuildable index backend and uses `.index/milvus/milvus.db` by - default for Milvus Lite. + as the same rebuildable index backend and lives outside the memory root in a + configured Milvus Server or Zilliz Cloud deployment. Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. When an episode is superseded by a Reflection merge, diff --git a/pyproject.toml b/pyproject.toml index 991d9c91b..7006dcd7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ otel = [ "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", ] +milvus = ["pymilvus>=3.0.0"] [project.urls] Homepage = "https://evermind.ai" @@ -249,6 +250,7 @@ ignore_imports = [ "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.tables", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.repos", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.lancedb_manager", + "everos.infra.persistence.index -> everos.infra.persistence.lancedb.predicate", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", @@ -280,4 +282,7 @@ dev = [ # [otel] stack is always present in the dev / CI environment. "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", + # Milvus tests must run in every dev/CI environment. The client package + # intentionally excludes the embedded milvus-lite extra. + "pymilvus>=3.0.0", ] diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index d67f0bc99..38d226823 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -57,15 +57,15 @@ cache_size_kb = 2048 backend = "lancedb" [milvus] -# Optional derived index backend. Leave uri empty to use Milvus Lite at -# /.index/milvus/milvus.db. Set uri/token/db_name for Milvus server or -# Zilliz Cloud. +# Optional remote derived index backend for Milvus Server or Zilliz Cloud. +# `uri` is required when [index] backend = "milvus". Embedded Milvus Lite and +# local database paths are not supported. Use a unique db_name or +# collection_prefix when multiple EverOS deployments share a cluster. # Override via EVEROS_MILVUS__URI, EVEROS_MILVUS__TOKEN, EVEROS_MILVUS__DB_NAME. uri = "" token = "" db_name = "" consistency_level = "Session" -dimension = 1024 collection_prefix = "everos" [llm] diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 5e36b390d..1e910ac8e 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -23,6 +23,7 @@ from __future__ import annotations import os +import re from functools import cache from pathlib import Path from typing import Literal @@ -392,12 +393,19 @@ class MilvusSettings(BaseModel): uri: str = "" token: SecretStr = SecretStr("") db_name: str = "" - consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = ( - "Session" - ) - dimension: int = Field(default=1024, ge=1) + consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = "Session" collection_prefix: str = Field(default="everos", min_length=1) + @field_validator("collection_prefix") + @classmethod + def _validate_collection_prefix(cls, value: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value): + raise ValueError( + "collection_prefix must start with a letter or underscore and " + "contain only letters, digits, and underscores" + ) + return value + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" diff --git a/src/everos/core/persistence/memory_root.py b/src/everos/core/persistence/memory_root.py index 5ca928c4f..701ff0012 100644 --- a/src/everos/core/persistence/memory_root.py +++ b/src/everos/core/persistence/memory_root.py @@ -11,7 +11,6 @@ .index/ derived indexes (rebuildable from markdown) sqlite/ system.db (+ WAL/SHM), ome.db, ome.aps.db lancedb/ LanceDB tables - milvus/ Milvus Lite database when Milvus is selected .tmp/ atomic-write staging directory .lock single-process lock anchor (created on demand by ``memory_root_lock``) @@ -166,16 +165,6 @@ def lancedb_dir(self) -> Path: """``/.index/lancedb/`` — LanceDB table root.""" return self.index_dir / "lancedb" - @property - def milvus_dir(self) -> Path: - """``/.index/milvus/`` — Milvus Lite derived index root.""" - return self.index_dir / "milvus" - - @property - def milvus_db(self) -> Path: - """``/.index/milvus/milvus.db`` — default Milvus Lite URI.""" - return self.milvus_dir / "milvus.db" - @property def sqlite_dir(self) -> Path: """``/.index/sqlite/`` — SQLite system DB root. @@ -255,12 +244,10 @@ def ensure(self) -> None: /.index/ /.index/sqlite/ /.index/lancedb/ - /.index/milvus/ /.tmp/ """ self.root.mkdir(parents=True, exist_ok=True) self.index_dir.mkdir(parents=True, exist_ok=True) self.sqlite_dir.mkdir(parents=True, exist_ok=True) self.lancedb_dir.mkdir(parents=True, exist_ok=True) - self.milvus_dir.mkdir(parents=True, exist_ok=True) self.tmp_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index d20e05b63..56994f565 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -1,4 +1,4 @@ -"""LanceDB lifespan provider (HTTP API entrypoint). +"""Derived-index lifespan provider (HTTP API entrypoint). Startup: Open the connection via ``get_connection`` (lazy, idempotent). @@ -31,13 +31,18 @@ from everos.core.lifespan import LifespanProvider from everos.core.observability.logging import get_logger +from everos.infra.persistence.index import ( + active_backend, +) +from everos.infra.persistence.index import ( + shutdown as shutdown_index, +) +from everos.infra.persistence.index import ( + startup as startup_index, +) from everos.infra.persistence.lancedb import ( BUSINESS_SCHEMAS_WITH_VECTOR, - dispose_connection, - ensure_business_indexes, - get_connection, get_table, - verify_business_schemas, ) logger = get_logger(__name__) @@ -83,7 +88,7 @@ async def _log_unbackfilled_hint() -> None: class LanceDBLifespanProvider(LifespanProvider): - """Manage the LanceDB connection + table cache for the app lifecycle. + """Manage the configured derived index for the app lifecycle. Startup runs four steps: @@ -100,12 +105,12 @@ def __init__(self, order: int = 11) -> None: super().__init__(name="lancedb", order=order) async def startup(self, app: FastAPI) -> Any: - conn = await get_connection() - await verify_business_schemas() - await ensure_business_indexes() - await _log_unbackfilled_hint() - logger.info("lancedb_ready", uri=conn.uri) + backend = active_backend() + conn = await startup_index() + if backend == "lancedb": + await _log_unbackfilled_hint() + logger.info("derived_index_ready", backend=backend) return conn async def shutdown(self, app: FastAPI) -> None: - await dispose_connection() + await shutdown_index() diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 8710f947b..d39f46944 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -15,7 +15,7 @@ vectors, build clusters, extract skills. See :func:`everos.entrypoints.cli.commands._backfill_cmd.run_backfill` for the phase orchestration. -- ``cascade rebuild`` — drop every business LanceDB table and re-index +- ``cascade rebuild`` — drop every business derived-index table and re-index all md from scratch. Recovery for a drifted / corrupt index; safe because md is the source of truth and un-extracted buffered messages are preserved. Skips the schema-verify guard (which the drift would @@ -46,13 +46,16 @@ from everos.core.persistence import MemoryRoot from everos.entrypoints.cli._log_setup import configure_cli_logging from everos.entrypoints.cli.commands._backfill_cmd import run_backfill -from everos.infra.persistence.lancedb import ( - dispose_connection, +from everos.infra.persistence.index import ( + active_backend, + connect, drop_business_tables, ensure_business_indexes, - get_connection, verify_business_schemas, ) +from everos.infra.persistence.index import ( + shutdown as shutdown_index, +) from everos.infra.persistence.sqlite import ( dispose_engine, get_engine, @@ -68,7 +71,7 @@ app = typer.Typer( name="cascade", - help="Inspect and operate the md → LanceDB sync queue", + help="Inspect and operate the markdown → derived-index sync queue", no_args_is_help=True, ) @@ -138,7 +141,7 @@ def _apply_verbose_logging(verbose: bool | None) -> None: async def _runtime( # type: ignore[no-untyped-def] *, verify: bool = True, ensure: bool = True ): - """Stand up sqlite + lancedb the same way the API lifespan would. + """Stand up SQLite + the configured index the way the API lifespan would. The CLI uses the same lazy, process-wide singletons the API lifespan does. They are **per-process**: a running daemon has its own @@ -163,15 +166,21 @@ async def _runtime( # type: ignore[no-untyped-def] engine = get_engine() async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) - await get_connection() - if verify: - await verify_business_schemas() - if ensure: - await ensure_business_indexes() + await connect() + if active_backend() == "milvus": + if ensure: + await ensure_business_indexes() + if verify: + await verify_business_schemas() + else: + if verify: + await verify_business_schemas() + if ensure: + await ensure_business_indexes() try: yield finally: - await dispose_connection() + await shutdown_index() await dispose_engine() @@ -451,7 +460,7 @@ def rebuild( typer.Option("--yes", "-y", help="Skip the confirmation prompt."), ] = False, ) -> None: - """Rebuild the LanceDB index from markdown (recover from schema drift). + """Rebuild the configured derived index from markdown. **Stop the ``everos server`` first** — this is the one cascade command that is not safe alongside a live daemon. It drops and recreates the @@ -459,13 +468,13 @@ def rebuild( the dropped dataset; the command refuses to start while a server holds the OME lock. - Drops every business LanceDB table and re-indexes all md from + Drops every business table or collection and re-indexes all markdown from scratch. Markdown is the source of truth, so no memory content is lost, and this is the safe recovery from a drifted / corrupt index (e.g. the ``verify_business_schemas`` startup failure): - - unlike ``rm -rf ~/.everos/.index/lancedb``, it re-populates - already-indexed entries (that command leaves the cascade queue + - unlike deleting ``~/.everos/.index/lancedb`` on the default backend, it + re-populates already-indexed entries (that command leaves the cascade queue marked ``done``, so nothing re-indexes and the index comes back empty); - unlike ``rm -rf ~/.everos/.index``, it preserves SQLite state that @@ -476,7 +485,7 @@ def rebuild( typer.echo( "error: a server (or another exclusive CLI phase) is running on " "this memory root.\n" - " cascade rebuild drops and recreates the LanceDB tables; a live " + " cascade rebuild drops and recreates the derived index; a live " "daemon holds cached\n" " table handles and would keep writing to the dropped dataset. " "Stop `everos server`\n" @@ -486,7 +495,7 @@ def rebuild( raise typer.Exit(code=3) if not yes: typer.confirm( - "Drop all LanceDB business tables and re-index from markdown? " + "Drop all derived-index business data and re-index from markdown? " "(requires the server to be stopped)", abort=True, ) @@ -507,7 +516,7 @@ async def _run() -> None: typer.echo(f"reset {cleared} cascade queue row(s)") dropped = await drop_business_tables() typer.echo( - f"dropped {len(dropped)} LanceDB table(s): " + f"dropped {len(dropped)} derived-index table(s): " f"{', '.join(dropped) or '(none)'}" ) # Recreate the tables (current schema) + FTS indexes. diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py index 0cc969eb6..dfe2bcab3 100644 --- a/src/everos/infra/persistence/index/__init__.py +++ b/src/everos/infra/persistence/index/__init__.py @@ -30,6 +30,23 @@ ParentType, UserProfile, ) +from everos.infra.persistence.lancedb.predicate import render_predicate as _render_lance + +from .predicate import ( + Predicate, + all_of, + any_of, + contains, + eq, + gt, + gte, + is_null, + lt, + lte, + ne, + one_of, +) +from .schema import schema_for def active_backend() -> str: @@ -50,6 +67,13 @@ async def startup() -> Any: return conn +async def connect() -> Any: + """Open the active backend without creating or verifying indexes.""" + if active_backend() == "milvus": + return await _milvus().get_client() + return await _lancedb.get_connection() + + async def shutdown() -> None: """Dispose the configured derived index backend.""" if active_backend() == "milvus": @@ -72,12 +96,20 @@ async def verify_business_schemas() -> None: await _lancedb.verify_business_schemas() +async def drop_business_tables() -> list[str]: + """Drop all business indexes for the active backend.""" + if active_backend() == "milvus": + return await _milvus().drop_business_tables() + return await _lancedb.drop_business_tables() + + class _LanceIndexRepoAdapter: """Add backend-neutral recall helpers to an existing LanceDB repo.""" def __init__(self, repo: Any, schema: type[Any]) -> None: self._repo = repo self.schema = schema + self.index_schema = schema_for(schema) def __getattr__(self, name: str) -> Any: return getattr(self._repo, name) @@ -90,7 +122,7 @@ async def sparse_search( columns: Sequence[str] | None = None, limit: int, ) -> list[dict[str, Any]]: - fields = list(columns or self.schema.BM25_FIELDS) + fields = list(columns or self.index_schema.bm25_fields) if not query_terms or not fields: return [] table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) @@ -125,6 +157,7 @@ async def dense_search( where: Any, *, limit: int, + vector_field: str = "vector", ) -> list[dict[str, Any]]: if not vector: return [] @@ -132,6 +165,7 @@ async def dense_search( return ( await table.query() .nearest_to(list(vector)) + .column(vector_field) .distance_type("cosine") .where(_lance_expr(where)) .limit(limit) @@ -148,10 +182,11 @@ def __init__( self._lance = _LanceIndexRepoAdapter(lance_repo, schema) self._milvus_repo_name = milvus_repo_name self.schema = schema + self.index_schema = schema_for(schema) @property def table_name(self) -> str: - return self.schema.TABLE_NAME + return self.index_schema.table_name def _repo(self) -> Any: if active_backend() == "milvus": @@ -240,21 +275,40 @@ async def sparse_search( ) async def dense_search( - self, vector: Sequence[float], where: Any, *, limit: int + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + vector_field: str = "vector", ) -> list[dict[str, Any]]: return await self._repo().dense_search( - vector, _where_for_backend(where), limit=limit + vector, + _where_for_backend(where), + limit=limit, + vector_field=vector_field, ) def _where_for_backend(where: Any) -> Any: + if where is None: + return None if active_backend() == "milvus" else "" + if not isinstance(where, Predicate): + raise TypeError( + "derived-index repository predicates must use the neutral " + f"Predicate AST, got {type(where).__name__}" + ) if active_backend() == "milvus": - return getattr(where, "milvus", where) - return _lance_expr(where) + return where + return _render_lance(where) def _lance_expr(where: Any) -> str: - return getattr(where, "lancedb", where) + if isinstance(where, Predicate): + return _render_lance(where) + if isinstance(where, str): + return where + raise TypeError(f"unsupported LanceDB predicate: {type(where).__name__}") def _build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery: @@ -311,15 +365,29 @@ def _milvus() -> Any: "Foresight", "KnowledgeTopic", "ParentType", + "Predicate", "UserProfile", "active_backend", "agent_case_repo", "agent_skill_repo", + "all_of", + "any_of", "atomic_fact_repo", + "connect", + "contains", + "drop_business_tables", "ensure_business_indexes", "episode_repo", + "eq", "foresight_repo", + "gt", + "gte", + "is_null", "knowledge_topic_repo", + "lt", + "lte", + "ne", + "one_of", "shutdown", "startup", "user_profile_repo", diff --git a/src/everos/infra/persistence/index/predicate.py b/src/everos/infra/persistence/index/predicate.py new file mode 100644 index 000000000..4892356eb --- /dev/null +++ b/src/everos/infra/persistence/index/predicate.py @@ -0,0 +1,45 @@ +"""Compatibility re-exports for the shared derived-index predicate AST.""" + +from everos.infra.persistence.predicate import All as All +from everos.infra.persistence.predicate import AnyOf as AnyOf +from everos.infra.persistence.predicate import Comparison as Comparison +from everos.infra.persistence.predicate import Contains as Contains +from everos.infra.persistence.predicate import In as In +from everos.infra.persistence.predicate import IsNull as IsNull +from everos.infra.persistence.predicate import Predicate as Predicate +from everos.infra.persistence.predicate import Scalar as Scalar +from everos.infra.persistence.predicate import all_of as all_of +from everos.infra.persistence.predicate import any_of as any_of +from everos.infra.persistence.predicate import compare as compare +from everos.infra.persistence.predicate import contains as contains +from everos.infra.persistence.predicate import eq as eq +from everos.infra.persistence.predicate import gt as gt +from everos.infra.persistence.predicate import gte as gte +from everos.infra.persistence.predicate import is_null as is_null +from everos.infra.persistence.predicate import lt as lt +from everos.infra.persistence.predicate import lte as lte +from everos.infra.persistence.predicate import ne as ne +from everos.infra.persistence.predicate import one_of as one_of + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/infra/persistence/index/schema.py b/src/everos/infra/persistence/index/schema.py new file mode 100644 index 000000000..dd9294baa --- /dev/null +++ b/src/everos/infra/persistence/index/schema.py @@ -0,0 +1,341 @@ +"""Backend-neutral schemas for rebuildable business indexes. + +Each table has one explicit field description consumed by every backend +adapter. The model-field parity check fails immediately when a domain field is +added without a storage decision, avoiding fallback coercions and silently +divergent schemas. +""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from enum import StrEnum +from functools import cache +from typing import Any, get_args, get_origin + +_DEFAULT_STRING_LENGTH = 65_535 +_ID_LENGTH = 512 +_ARRAY_CAPACITY = 256 +_VECTOR_DIMENSION = 1024 + + +class IndexFieldKind(StrEnum): + STRING = "string" + STRING_ARRAY = "string_array" + FLOAT = "float" + INTEGER = "integer" + DATETIME = "datetime" + DENSE_VECTOR = "dense_vector" + + +@dataclass(frozen=True) +class IndexField: + name: str + kind: IndexFieldKind + nullable: bool = False + primary: bool = False + max_length: int | None = None + max_capacity: int | None = None + dimension: int | None = None + + +@dataclass(frozen=True) +class IndexSchema: + table_name: str + model: type[Any] + fields: tuple[IndexField, ...] + bm25_fields: tuple[str, ...] + + def field(self, name: str) -> IndexField: + for field in self.fields: + if field.name == name: + return field + raise KeyError(name) + + @property + def vector_fields(self) -> tuple[IndexField, ...]: + return tuple( + field for field in self.fields if field.kind is IndexFieldKind.DENSE_VECTOR + ) + + @property + def datetime_fields(self) -> frozenset[str]: + return frozenset( + field.name for field in self.fields if field.kind is IndexFieldKind.DATETIME + ) + + +def _s( + name: str, + *, + nullable: bool = False, + primary: bool = False, + max_length: int = _DEFAULT_STRING_LENGTH, +) -> IndexField: + return IndexField( + name, + IndexFieldKind.STRING, + nullable=nullable, + primary=primary, + max_length=max_length, + ) + + +def _id(name: str = "id") -> IndexField: + return _s(name, primary=True, max_length=_ID_LENGTH) + + +def _a(name: str, *, nullable: bool = False) -> IndexField: + return IndexField( + name, + IndexFieldKind.STRING_ARRAY, + nullable=nullable, + max_length=_ID_LENGTH, + max_capacity=_ARRAY_CAPACITY, + ) + + +def _f(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.FLOAT, nullable=nullable) + + +def _i(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.INTEGER, nullable=nullable) + + +def _d(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.DATETIME, nullable=nullable) + + +def _v(name: str, *, nullable: bool = True) -> IndexField: + return IndexField( + name, + IndexFieldKind.DENSE_VECTOR, + nullable=nullable, + dimension=_VECTOR_DIMENSION, + ) + + +_FIELDS: dict[str, tuple[IndexField, ...]] = { + "episode": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("subject", nullable=True), + _s("summary", nullable=True), + _s("episode"), + _s("episode_tokens"), + _s("md_path"), + _s("content_sha256"), + _s("deprecated_by", nullable=True), + _v("vector"), + _v("subject_vector"), + _d("created_at"), + _d("updated_at"), + ), + "atomic_fact": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("fact"), + _s("fact_tokens"), + _s("md_path"), + _s("content_sha256"), + _s("deprecated_by", nullable=True), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "foresight": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _d("start_time", nullable=True), + _d("end_time", nullable=True), + _i("duration_days", nullable=True), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("foresight"), + _s("foresight_tokens"), + _s("evidence", nullable=True), + _s("evidence_tokens", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "agent_case": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id"), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _f("quality_score"), + _s("task_intent"), + _s("task_intent_tokens"), + _s("approach"), + _s("approach_tokens"), + _s("key_insight", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "agent_skill": ( + _id(), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("name"), + _s("description"), + _s("description_tokens"), + _s("content"), + _s("content_tokens"), + _f("confidence"), + _f("maturity_score"), + _a("source_case_ids"), + _s("cluster_id", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "user_profile": ( + _id(), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("summary"), + _s("explicit_info_json"), + _s("implicit_traits_json"), + _i("profile_timestamp_ms"), + _s("md_path"), + _s("content_sha256"), + _d("created_at"), + _d("updated_at"), + ), + "knowledge_topic": ( + _id(), + _s("doc_id"), + _s("category_id"), + _s("app_id"), + _s("project_id"), + _s("topic_name"), + _s("topic_path"), + _i("depth"), + _s("parent_node_id"), + _s("summary"), + _s("summary_tokens"), + _s("content_tokens"), + _a("content_labels"), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), +} + + +@cache +def schema_for(model: type[Any]) -> IndexSchema: + """Return and validate the explicit neutral schema for a model.""" + table_name = model.TABLE_NAME + try: + fields = _FIELDS[table_name] + except KeyError as exc: + raise ValueError(f"no derived-index schema for {table_name!r}") from exc + + declared = {field.name for field in fields} + actual = set(model.model_fields) + if declared != actual: + raise ValueError( + f"derived-index schema drift for {table_name!r}: " + f"missing={sorted(actual - declared)}, stale={sorted(declared - actual)}" + ) + for field in fields: + _validate_model_field(table_name, field, model.model_fields[field.name]) + bm25_fields = tuple(model.BM25_FIELDS) + unknown_bm25 = set(bm25_fields) - declared + if unknown_bm25: + raise ValueError( + f"derived-index schema {table_name!r} has unknown BM25 fields: " + f"{sorted(unknown_bm25)}" + ) + return IndexSchema(table_name, model, fields, bm25_fields) + + +def _validate_model_field(table_name: str, field: IndexField, model_field: Any) -> None: + annotation = model_field.annotation + args = get_args(annotation) + optional = type(None) in args + candidates = ( + tuple(arg for arg in args if arg is not type(None)) + if optional + else (annotation,) + ) + if optional != field.nullable: + raise ValueError( + f"derived-index schema {table_name}.{field.name} nullable drift: " + f"model={optional}, schema={field.nullable}" + ) + + valid = False + if field.kind is IndexFieldKind.STRING: + valid = candidates == (str,) + elif field.kind is IndexFieldKind.STRING_ARRAY: + valid = len(candidates) == 1 and ( + get_origin(candidates[0]) is list and get_args(candidates[0]) == (str,) + ) + elif field.kind is IndexFieldKind.FLOAT: + valid = candidates == (float,) + elif field.kind is IndexFieldKind.INTEGER: + valid = candidates == (int,) + elif field.kind is IndexFieldKind.DATETIME: + valid = candidates == (dt.datetime,) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + dimension = getattr(candidates[0], "dim", None) if candidates else None + if callable(dimension): + dimension = dimension() + valid = len(candidates) == 1 and dimension == field.dimension + if not valid: + raise ValueError( + f"derived-index schema {table_name}.{field.name} type drift: " + f"model={annotation!r}, schema={field.kind.value}" + ) + + +__all__ = ["IndexField", "IndexFieldKind", "IndexSchema", "schema_for"] diff --git a/src/everos/infra/persistence/lancedb/predicate.py b/src/everos/infra/persistence/lancedb/predicate.py new file mode 100644 index 000000000..8a429d57e --- /dev/null +++ b/src/everos/infra/persistence/lancedb/predicate.py @@ -0,0 +1,83 @@ +"""Render backend-neutral predicates as LanceDB DataFusion expressions.""" + +from __future__ import annotations + +import datetime as dt +import re +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_iso_format +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate(predicate: Predicate | None) -> str: + """Render a predicate with LanceDB escaping and timestamp literals.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field)} {_OPERATORS[predicate.operator]} " + f"{_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field)} IN ({values})" + if isinstance(predicate, Contains): + return f"array_has({_field(predicate.field)}, {_literal(predicate.value)})" + if isinstance(predicate, IsNull): + return f"{_field(predicate.field)} IS NULL" + if isinstance(predicate, All): + return _render_group(predicate.children, "AND") + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "OR") + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group(children: tuple[Predicate, ...], operator: str) -> str: + rendered = [render_predicate(child) for child in children] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return f"'{value.replace(chr(39), chr(39) * 2)}'" + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return f"TIMESTAMP '{to_iso_format(aware)}'" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py index 4396360c8..910f8e64b 100644 --- a/src/everos/infra/persistence/milvus/__init__.py +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -7,6 +7,11 @@ from __future__ import annotations +import asyncio + +from everos.core.observability.logging import get_logger + +from .milvus_manager import MilvusConfigurationError as MilvusConfigurationError from .milvus_manager import MilvusSchemaMismatchError as MilvusSchemaMismatchError from .milvus_manager import dispose_connection as dispose_connection from .milvus_manager import get_client as get_client @@ -18,6 +23,10 @@ from .repos import foresight_repo as foresight_repo from .repos import knowledge_topic_repo as knowledge_topic_repo from .repos import user_profile_repo as user_profile_repo +from .repository import MilvusRepoBase as MilvusRepoBase +from .repository import MilvusValueLimitError as MilvusValueLimitError + +logger = get_logger(__name__) async def ensure_business_indexes() -> None: @@ -32,13 +41,40 @@ async def verify_business_schemas() -> None: await repo.verify_collection() +async def drop_business_tables() -> list[str]: + """Drop every configured Milvus collection and return their names.""" + client = await get_client() + dropped: list[str] = [] + for repo in ALL_REPOS: + name = repo.collection_name + if await asyncio.to_thread(client.has_collection, name): + try: + await asyncio.to_thread(client.drop_collection, name) + except Exception: + # Zilliz Serverless can complete the drop server-side while + # its gateway returns DEADLINE_EXCEEDED. Confirm state before + # turning an already-successful rebuild/cleanup into failure. + if await asyncio.to_thread(client.has_collection, name): + raise + logger.warning( + "milvus_collection_drop_confirmed_after_client_error", + collection=name, + ) + dropped.append(name) + MilvusRepoBase._reset_collection_cache() + return dropped + + __all__ = [ "ALL_REPOS", + "MilvusConfigurationError", "MilvusSchemaMismatchError", + "MilvusValueLimitError", "agent_case_repo", "agent_skill_repo", "atomic_fact_repo", "dispose_connection", + "drop_business_tables", "ensure_business_indexes", "episode_repo", "foresight_repo", diff --git a/src/everos/infra/persistence/milvus/milvus_manager.py b/src/everos/infra/persistence/milvus/milvus_manager.py index 602e4a314..05af230b4 100644 --- a/src/everos/infra/persistence/milvus/milvus_manager.py +++ b/src/everos/infra/persistence/milvus/milvus_manager.py @@ -7,8 +7,8 @@ from pymilvus import MilvusClient from everos.config import MilvusSettings, load_settings +from everos.core.errors import ConfigurationError from everos.core.observability.logging import get_logger -from everos.core.persistence import MemoryRoot logger = get_logger(__name__) @@ -19,6 +19,10 @@ class MilvusSchemaMismatchError(RuntimeError): """Raised when an existing Milvus collection does not match EverOS.""" +class MilvusConfigurationError(ConfigurationError): + """Raised when the remote Milvus profile is incomplete or invalid.""" + + def collection_name(table_name: str, settings: MilvusSettings | None = None) -> str: """Return the configured Milvus collection name for an EverOS table.""" cfg = settings or load_settings().milvus @@ -55,14 +59,27 @@ async def dispose_connection() -> None: _client.close() _client = None logger.info("milvus_connection_closed") + from .repository import MilvusRepoBase + + MilvusRepoBase._reset_collection_cache() def _resolve_uri(settings: MilvusSettings) -> str: - if settings.uri: - return settings.uri - memory_root = MemoryRoot.default() - memory_root.milvus_dir.mkdir(parents=True, exist_ok=True) - return str(memory_root.milvus_db) + uri = settings.uri.strip() + if not uri: + raise MilvusConfigurationError( + "[index] backend = 'milvus' requires EVEROS_MILVUS__URI (or " + "[milvus] uri) pointing to Milvus Server or Zilliz Cloud; " + "embedded Milvus Lite is not supported" + ) + scheme, separator, _rest = uri.partition("://") + if not separator or scheme.lower() not in {"http", "https"}: + raise MilvusConfigurationError( + "[milvus] uri must be a remote http(s) endpoint for Milvus Server " + "or Zilliz Cloud, not a local database path; embedded Milvus Lite " + "is not supported" + ) + return uri def _secret(value: object | None) -> str: @@ -80,6 +97,7 @@ def _sanitize_name_part(value: str) -> str: __all__ = [ + "MilvusConfigurationError", "MilvusSchemaMismatchError", "collection_name", "dispose_connection", diff --git a/src/everos/infra/persistence/milvus/predicate.py b/src/everos/infra/persistence/milvus/predicate.py new file mode 100644 index 000000000..37f14b289 --- /dev/null +++ b/src/everos/infra/persistence/milvus/predicate.py @@ -0,0 +1,100 @@ +"""Render backend-neutral predicates as Milvus filter expressions.""" + +from __future__ import annotations + +import datetime as dt +import json +import re +from collections.abc import Collection +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_timestamp_ms +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate( + predicate: Predicate | None, + *, + datetime_fields: Collection[str] = (), +) -> str: + """Render a predicate using Milvus operators and physical field names.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field, datetime_fields)} " + f"{_OPERATORS[predicate.operator]} {_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field, datetime_fields)} in [{values}]" + if isinstance(predicate, Contains): + return ( + f"array_contains({_field(predicate.field, datetime_fields)}, " + f"{_literal(predicate.value)})" + ) + if isinstance(predicate, IsNull): + return f"{_field(predicate.field, datetime_fields)} is null" + if isinstance(predicate, All): + return _render_group(predicate.children, "and", datetime_fields) + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "or", datetime_fields) + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group( + children: tuple[Predicate, ...], + operator: str, + datetime_fields: Collection[str], +) -> str: + rendered = [ + render_predicate(child, datetime_fields=datetime_fields) for child in children + ] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str, datetime_fields: Collection[str]) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + if value in datetime_fields: + return f"{value}_ms" + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return str(to_timestamp_ms(aware)) + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py index 826bc9267..9319a9d9e 100644 --- a/src/everos/infra/persistence/milvus/repos.py +++ b/src/everos/infra/persistence/milvus/repos.py @@ -3,7 +3,10 @@ from __future__ import annotations from collections.abc import Sequence +from typing import Any +from everos.component.utils.datetime import from_timestamp +from everos.infra.persistence.index.predicate import all_of, eq, gt, is_null from everos.infra.persistence.lancedb import ( AgentCase, AgentSkill, @@ -20,6 +23,50 @@ class _EpisodeRepo(MilvusRepoBase[Episode]): schema = Episode + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await self._count_where( + all_of( + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + eq("parent_type", parent_type) if parent_type is not None else None, + ) + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Episode] | list[dict[str, Any]]: + predicate = all_of( + eq("owner_id", owner_id), + gt("timestamp", from_timestamp(after_ts)), + eq("parent_type", parent_type), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + ) + rows = await self.find_where(predicate, limit=limit or 20_000) + rows.sort(key=lambda row: row.timestamp) + if columns is None: + return rows + projection = list(dict.fromkeys([*columns, "timestamp"])) + return [{name: getattr(row, name) for name in projection} for row in rows] + class _AtomicFactRepo(MilvusRepoBase[AtomicFact]): schema = AtomicFact @@ -37,17 +84,15 @@ class _AgentSkillRepo(MilvusRepoBase[AgentSkill]): schema = AgentSkill async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: - rows = await self.find_where( - f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", - limit=10_000, + return await self._count_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)) ) - return len(rows) async def find_in_cluster( self, *, owner_id: str, cluster_id: str, limit: int ) -> list[AgentSkill]: return await self.find_where( - f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), limit=limit, ) @@ -66,7 +111,7 @@ async def find_topk_relevant_in_cluster( ) rows = await self.dense_search( query_vector, - f"owner_id = '{_q(owner_id)}' AND cluster_id = '{_q(cluster_id)}'", + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), limit=top_k, ) out: list[AgentSkill] = [] @@ -85,10 +130,6 @@ class _KnowledgeTopicRepo(MilvusRepoBase[KnowledgeTopic]): schema = KnowledgeTopic -def _q(value: str) -> str: - return value.replace("\\", "\\\\").replace("'", "\\'") - - episode_repo = _EpisodeRepo() atomic_fact_repo = _AtomicFactRepo() foresight_repo = _ForesightRepo() diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py index de9143b3d..44eaa11f5 100644 --- a/src/everos/infra/persistence/milvus/repository.py +++ b/src/everos/infra/persistence/milvus/repository.py @@ -1,55 +1,62 @@ -"""Milvus repository implementation for EverOS derived index tables.""" +"""Milvus repository for EverOS rebuildable derived indexes.""" from __future__ import annotations import asyncio import datetime as dt -import re from collections.abc import Sequence -from importlib.metadata import PackageNotFoundError, version -from typing import Any, ClassVar, get_args, get_origin +from typing import Any, ClassVar +from pydantic import BaseModel from pymilvus import DataType, Function, FunctionType, MilvusClient -from everos.component.utils.datetime import ( - ensure_utc, - from_iso_format, - from_timestamp, - to_timestamp_ms, -) +from everos.component.utils.datetime import ensure_utc, from_timestamp, to_timestamp_ms from everos.config import load_settings from everos.core.observability.logging import get_logger -from everos.core.persistence import BaseLanceTable - -from .milvus_manager import ( - MilvusSchemaMismatchError, - collection_name, - get_client, +from everos.infra.persistence.index.predicate import ( + Predicate, + all_of, + eq, + one_of, ) +from everos.infra.persistence.index.schema import ( + IndexField, + IndexFieldKind, + IndexSchema, + schema_for, +) + +from .milvus_manager import MilvusSchemaMismatchError, collection_name, get_client +from .predicate import render_predicate logger = get_logger(__name__) _DUMMY_VECTOR_FIELD = "_everos_dummy_vector" +_DUMMY_VECTOR_DIMENSION = 2 _SPARSE_SUFFIX = "__sparse" -_MAX_VARCHAR_LENGTH = 65_535 -_ID_MAX_LENGTH = 512 -_ARRAY_MAX_CAPACITY = 256 +_PRESENT_SUFFIX = "__present" +_UPDATE_FETCH_LIMIT = 10_000 -def _q(value: str) -> str: - """Escape a string for a Milvus single-quoted expression literal.""" - return value.replace("\\", "\\\\").replace("'", "\\'") +class MilvusValueLimitError(ValueError): + """A row exceeds a documented Milvus VARCHAR, array, or vector limit.""" -class MilvusRepoBase[T: BaseLanceTable]: - """Generic Milvus repository for one EverOS derived index collection.""" +class MilvusRepoBase[T: BaseModel]: + """Generic Milvus repository backed by one neutral index schema.""" schema: type[T] _write_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _collection_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _ready_collections: ClassVar[set[str]] = set() + + @property + def index_schema(self) -> IndexSchema: + return schema_for(self.schema) @property def table_name(self) -> str: - return self.schema.TABLE_NAME + return self.index_schema.table_name @property def collection_name(self) -> str: @@ -59,37 +66,58 @@ def collection_name(self) -> str: def _write_lock(cls, name: str) -> asyncio.Lock: return cls._write_locks.setdefault(name, asyncio.Lock()) + @classmethod + def _collection_lock(cls, name: str) -> asyncio.Lock: + return cls._collection_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_collection_cache(cls) -> None: + cls._ready_collections.clear() + cls._collection_locks.clear() + @classmethod def _reset_locks_for_tests(cls) -> None: cls._write_locks.clear() + cls._reset_collection_cache() async def ensure_collection(self) -> None: - client = await get_client() - if await _run(client.has_collection, self.collection_name): - await self.verify_collection() + """Create or verify the collection once per process.""" + name = self.collection_name + if name in self._ready_collections: return + async with self._collection_lock(name): + if name in self._ready_collections: + return + client = await get_client() + if await _run(client.has_collection, name): + await self.verify_collection() + else: + await self._create_collection(client) + self._ready_collections.add(name) + async def _create_collection(self, client: MilvusClient) -> None: schema = self._build_collection_schema() index_params = client.prepare_index_params() - if "vector" in self.schema.model_fields: - index_params.add_index( - field_name="vector", - index_type="AUTOINDEX", - metric_type="COSINE", - ) + vector_fields = self.index_schema.vector_fields + if vector_fields: + for field in vector_fields: + index_params.add_index( + field_name=field.name, + index_type="AUTOINDEX", + metric_type="COSINE", + ) else: index_params.add_index( field_name=_DUMMY_VECTOR_FIELD, index_type="AUTOINDEX", metric_type="COSINE", ) - for field in self.schema.BM25_FIELDS: + for field in self.index_schema.bm25_fields: index_params.add_index( field_name=_sparse_field(field), index_type="AUTOINDEX", metric_type="BM25", ) - settings = load_settings().milvus await _run( client.create_collection, @@ -110,21 +138,20 @@ async def verify_collection(self) -> None: actual = {field["name"] for field in description.get("fields", [])} expected = set(self._stored_field_names()) missing = expected - actual - if missing: + stale = actual - expected + if missing or stale: raise MilvusSchemaMismatchError( f"Milvus collection {self.collection_name!r} schema drift: " - f"missing={sorted(missing)}. The index is rebuildable from md; " - "drop the collection and restart to rebuild it." + f"missing={sorted(missing)}, stale={sorted(stale)}. The index is " + "rebuildable from markdown; run `everos cascade rebuild`." ) - # ── Create / update ───────────────────────────────────────────── - async def add(self, records: Sequence[T]) -> None: if not records: return await self.ensure_collection() - client = await get_client() payload = [self._to_milvus_record(record) for record in records] + client = await get_client() async with self._write_lock(self.collection_name): await _run(client.insert, self.collection_name, payload) @@ -134,52 +161,57 @@ async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: if not records: return await self.ensure_collection() - client = await get_client() payload = [self._to_milvus_record(record) for record in records] + client = await get_client() async with self._write_lock(self.collection_name): await _run(client.upsert, self.collection_name, payload) - async def update(self, updates: dict[str, Any], *, where: Any) -> None: - await self.ensure_collection() - rows = await self._query_raw(where, limit=10_000, include_vectors=True) + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + rows = await self._query_raw( + where, limit=_UPDATE_FETCH_LIMIT, include_vectors=True + ) if not rows: return - patched = [] + if len(rows) == _UPDATE_FETCH_LIMIT: + logger.warning( + "milvus_update_truncated", + table=self.table_name, + limit=_UPDATE_FETCH_LIMIT, + ) + patched: list[dict[str, Any]] = [] for row in rows: merged = dict(row) for key, value in updates.items(): self._write_field_value(merged, key, value) + self._validate_raw_record(merged) patched.append(merged) client = await get_client() async with self._write_lock(self.collection_name): await _run(client.upsert, self.collection_name, patched) - # ── Maintenance ──────────────────────────────────────────────── - async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: - """Milvus indexes are managed by the service; no per-write compaction.""" + """Milvus indexes and compaction are service-managed.""" async def rebuild_indexes(self) -> None: """Milvus AUTOINDEX maintenance is service-managed.""" - # ── Reads ────────────────────────────────────────────────────── - async def count(self) -> int: + return await self._count_where(None) + + async def _count_where(self, where: Predicate | None) -> int: await self.ensure_collection() client = await get_client() rows = await _run( client.query, self.collection_name, - filter="", + filter=self._expr(where), output_fields=["count(*)"], ) - if not rows: - return 0 - return int(rows[0].get("count(*)", 0)) + return int(rows[0].get("count(*)", 0)) if rows else 0 async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: if id_field != "id": - rows = await self.find_where(f"{id_field} = '{_q(id_value)}'", limit=1) + rows = await self.find_where(eq(id_field, id_value), limit=1) return rows[0] if rows else None await self.ensure_collection() client = await get_client() @@ -189,21 +221,19 @@ async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: ids=[id_value], output_fields=self._output_fields(include_vectors=True), ) - if not rows: - return None - return self._model_from_milvus(rows[0]) + return self._model_from_milvus(rows[0]) if rows else None - async def find_where(self, where: Any, *, limit: int = 100) -> list[T]: + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: rows = await self._query_raw(where, limit=limit, include_vectors=True) return [self._model_from_milvus(row) for row in rows] - async def find_one_where(self, where: Any) -> T | None: + async def find_one_where(self, where: Predicate) -> T | None: rows = await self.find_where(where, limit=1) return rows[0] if rows else None async def find_where_paginated( self, - where: Any, + where: Predicate, *, sort_by: str, descending: bool = True, @@ -211,9 +241,9 @@ async def find_where_paginated( page_size: int = 20, max_fetch: int = 20_000, ) -> tuple[list[T], int]: + total = await self._count_where(where) raw = await self._query_raw(where, limit=max_fetch, include_vectors=True) - total = len(raw) - if total >= max_fetch: + if total > len(raw): logger.warning( "milvus_find_where_paginated_truncated", table=self.table_name, @@ -229,16 +259,16 @@ async def find_where_paginated( return rows[offset : offset + page_size], total async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: - return await self.find_where(f"owner_id = '{_q(owner_id)}'", limit=limit) + return await self.find_where(eq("owner_id", owner_id), limit=limit) async def find_by_md_path(self, md_path: str) -> T | None: - return await self.find_one_where(f"md_path = '{_q(md_path)}'") + return await self.find_one_where(eq("md_path", md_path)) async def search( self, *, vector: Sequence[float] | None = None, - where: Any = None, + where: Predicate | None = None, limit: int = 10, ) -> list[dict[str, Any]]: if vector is None: @@ -248,7 +278,7 @@ async def search( async def sparse_search( self, query_terms: Sequence[str], - where: Any, + where: Predicate | None, *, columns: Sequence[str] | None = None, limit: int, @@ -256,11 +286,13 @@ async def sparse_search( if not query_terms: return [] await self.ensure_collection() - fields = list(columns or self.schema.BM25_FIELDS) + fields = list(columns or self.index_schema.bm25_fields) if not fields: return [] + unknown = set(fields) - set(self.index_schema.bm25_fields) + if unknown: + raise ValueError(f"unknown BM25 fields: {sorted(unknown)}") client = await get_client() - expr = self._expr(where) query = " ".join(term for term in query_terms if term) best: dict[str, dict[str, Any]] = {} for field in fields: @@ -269,7 +301,7 @@ async def sparse_search( self.collection_name, data=[query], anns_field=_sparse_field(field), - filter=expr, + filter=self._expr(where), limit=limit, output_fields=self._output_fields(include_vectors=False), search_params={"metric_type": "BM25"}, @@ -291,20 +323,26 @@ async def sparse_search( async def dense_search( self, vector: Sequence[float], - where: Any, + where: Predicate | None, *, limit: int, + vector_field: str = "vector", ) -> list[dict[str, Any]]: - if not vector or "vector" not in self.schema.model_fields: + if not vector: return [] + field = self.index_schema.field(vector_field) + if field.kind is not IndexFieldKind.DENSE_VECTOR: + raise ValueError(f"{vector_field!r} is not a dense-vector field") + self._validate_vector(field, vector) await self.ensure_collection() client = await get_client() + present = eq(_present_field(vector_field), True) results = await _run( client.search, self.collection_name, data=[list(vector)], - anns_field="vector", - filter=self._expr(where), + anns_field=vector_field, + filter=self._expr(all_of(where, present)), limit=limit, output_fields=self._output_fields(include_vectors=False), search_params={"metric_type": "COSINE"}, @@ -314,9 +352,7 @@ async def dense_search( for row in _first_result_set(results) ] - # ── Deletes ──────────────────────────────────────────────────── - - async def delete(self, predicate: Any) -> None: + async def delete(self, predicate: Predicate) -> None: await self.ensure_collection() client = await get_client() async with self._write_lock(self.collection_name): @@ -327,14 +363,11 @@ async def delete(self, predicate: Any) -> None: ) async def delete_by_md_path(self, md_path: str) -> int: - await self.ensure_collection() - before = await self.find_where(f"md_path = '{_q(md_path)}'", limit=10_000) - if not before: - return 0 - await self.delete(f"md_path = '{_q(md_path)}'") - return len(before) - - # ── Daily-log helpers ────────────────────────────────────────── + predicate = eq("md_path", md_path) + count = await self._count_where(predicate) + if count: + await self.delete(predicate) + return count async def find_by_owner_entry( self, @@ -345,8 +378,12 @@ async def find_by_owner_entry( project_id: str = "default", ) -> T | None: return await self.find_one_where( - f"owner_id = '{_q(owner_id)}' AND entry_id = '{_q(entry_id)}' " - f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'" + all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), + ) ) async def find_by_owner_entries( @@ -359,10 +396,13 @@ async def find_by_owner_entries( ) -> list[T]: if not entry_ids: return [] - quoted = ", ".join(f"'{_q(entry_id)}'" for entry_id in entry_ids) return await self.find_where( - f"owner_id = '{_q(owner_id)}' AND entry_id IN ({quoted}) " - f"AND app_id = '{_q(app_id)}' AND project_id = '{_q(project_id)}'", + all_of( + eq("owner_id", owner_id), + one_of("entry_id", list(entry_ids)), + eq("app_id", app_id), + eq("project_id", project_id), + ), limit=len(entry_ids), ) @@ -370,7 +410,7 @@ async def find_by_session( self, owner_id: str, session_id: str, *, limit: int = 100 ) -> list[T]: return await self.find_where( - f"owner_id = '{_q(owner_id)}' AND session_id = '{_q(session_id)}'", + all_of(eq("owner_id", owner_id), eq("session_id", session_id)), limit=limit, ) @@ -378,82 +418,24 @@ async def find_by_parent( self, parent_type: str, parent_id: str, *, limit: int = 100 ) -> list[T]: return await self.find_where( - f"parent_type = '{_q(parent_type)}' AND parent_id = '{_q(parent_id)}'", + all_of(eq("parent_type", parent_type), eq("parent_id", parent_id)), limit=limit, ) - # ── Field conversion ─────────────────────────────────────────── - def _build_collection_schema(self): # type: ignore[no-untyped-def] - settings = load_settings().milvus schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) - has_vector = False - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - annotation = field.annotation - if name == "id": - schema.add_field( - field_name=name, - datatype=DataType.VARCHAR, - is_primary=True, - max_length=_ID_MAX_LENGTH, - ) - elif name == "vector": - has_vector = True - schema.add_field( - field_name=name, - datatype=DataType.FLOAT_VECTOR, - dim=settings.dimension, - ) - elif _is_datetime(annotation): - schema.add_field( - field_name=_datetime_storage_field(name), - datatype=DataType.INT64, - nullable=_is_optional(annotation), - ) - elif _is_list(annotation): - schema.add_field( - field_name=name, - datatype=DataType.ARRAY, - element_type=DataType.VARCHAR, - max_capacity=_ARRAY_MAX_CAPACITY, - max_length=_ID_MAX_LENGTH, - nullable=_is_optional(annotation), - ) - elif _is_float(annotation): - schema.add_field( - field_name=name, - datatype=DataType.DOUBLE, - nullable=_is_optional(annotation), - ) - elif _is_int(annotation): - schema.add_field( - field_name=name, - datatype=DataType.INT64, - nullable=_is_optional(annotation), - ) - else: - kwargs: dict[str, Any] = { - "field_name": name, - "datatype": DataType.VARCHAR, - "max_length": _MAX_VARCHAR_LENGTH, - } - if name in self.schema.BM25_FIELDS: - kwargs["enable_analyzer"] = True - elif _is_optional(annotation): - kwargs["nullable"] = True - schema.add_field(**kwargs) - - if not has_vector: + for field in self.index_schema.fields: + self._add_schema_field(schema, field) + if not self.index_schema.vector_fields: schema.add_field( field_name=_DUMMY_VECTOR_FIELD, datatype=DataType.FLOAT_VECTOR, - dim=1, + dim=_DUMMY_VECTOR_DIMENSION, ) - for field in self.schema.BM25_FIELDS: + for field in self.index_schema.bm25_fields: + sparse_name = _sparse_field(field) schema.add_field( - field_name=_sparse_field(field), + field_name=sparse_name, datatype=DataType.SPARSE_FLOAT_VECTOR, ) schema.add_function( @@ -461,35 +443,91 @@ def _build_collection_schema(self): # type: ignore[no-untyped-def] name=f"{field}_bm25", function_type=FunctionType.BM25, input_field_names=[field], - output_field_names=[_sparse_field(field)], + output_field_names=[sparse_name], ) ) return schema + def _add_schema_field(self, schema: Any, field: IndexField) -> None: + if field.kind is IndexFieldKind.STRING: + kwargs: dict[str, Any] = { + "field_name": field.name, + "datatype": DataType.VARCHAR, + "max_length": field.max_length, + } + if field.primary: + kwargs["is_primary"] = True + elif field.nullable and field.name not in self.index_schema.bm25_fields: + kwargs["nullable"] = True + if field.name in self.index_schema.bm25_fields: + kwargs["enable_analyzer"] = True + schema.add_field(**kwargs) + elif field.kind is IndexFieldKind.STRING_ARRAY: + schema.add_field( + field_name=field.name, + datatype=DataType.ARRAY, + element_type=DataType.VARCHAR, + max_capacity=field.max_capacity, + max_length=field.max_length, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.FLOAT: + schema.add_field( + field_name=field.name, + datatype=DataType.DOUBLE, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.INTEGER: + schema.add_field( + field_name=field.name, + datatype=DataType.INT64, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.DATETIME: + schema.add_field( + field_name=_datetime_storage_field(field.name), + datatype=DataType.INT64, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + schema.add_field( + field_name=field.name, + datatype=DataType.FLOAT_VECTOR, + dim=field.dimension, + ) + schema.add_field( + field_name=_present_field(field.name), + datatype=DataType.BOOL, + ) + else: # pragma: no cover - enum exhaustiveness guard + raise TypeError(f"unsupported index field kind: {field.kind}") + def _stored_field_names(self) -> list[str]: names: list[str] = [] - has_vector = False - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - if name == "vector": - has_vector = True - names.append(name) - elif _is_datetime(field.annotation): - names.append(_datetime_storage_field(name)) + for field in self.index_schema.fields: + if field.kind is IndexFieldKind.DATETIME: + names.append(_datetime_storage_field(field.name)) else: - names.append(name) - if not has_vector: + names.append(field.name) + if field.kind is IndexFieldKind.DENSE_VECTOR: + names.append(_present_field(field.name)) + if not self.index_schema.vector_fields: names.append(_DUMMY_VECTOR_FIELD) - names.extend(_sparse_field(field) for field in self.schema.BM25_FIELDS) + names.extend(_sparse_field(field) for field in self.index_schema.bm25_fields) return names def _output_fields(self, *, include_vectors: bool) -> list[str]: fields: list[str] = [] + vector_names = {field.name for field in self.index_schema.vector_fields} + present_names = {_present_field(name) for name in vector_names} for name in self._stored_field_names(): - if name.endswith(_SPARSE_SUFFIX) or name == _DUMMY_VECTOR_FIELD: + if name.endswith(_SPARSE_SUFFIX): + continue + if name == _DUMMY_VECTOR_FIELD: + if include_vectors: + fields.append(name) continue - if not include_vectors and name == "vector": + if not include_vectors and (name in vector_names or name in present_names): continue fields.append(name) return fields @@ -497,35 +535,80 @@ def _output_fields(self, *, include_vectors: bool) -> list[str]: def _to_milvus_record(self, record: T) -> dict[str, Any]: raw = record.model_dump(mode="python") out: dict[str, Any] = {} - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - value = raw.get(name) - if name in self.schema.BM25_FIELDS and value is None: + for field in self.index_schema.fields: + value = raw.get(field.name) + if field.name in self.index_schema.bm25_fields and value is None: value = "" - if _is_datetime(field.annotation): - storage_name = _datetime_storage_field(name) - out[storage_name] = ( + if field.kind is IndexFieldKind.DATETIME: + out[_datetime_storage_field(field.name)] = ( _datetime_to_ms(value) if value is not None else None ) - elif name == "vector": - out[name] = list(value or []) - elif _is_list(field.annotation): - out[name] = [str(item) for item in (value or [])] + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + out[field.name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + out[_present_field(field.name)] = present + elif field.kind is IndexFieldKind.STRING_ARRAY: + out[field.name] = [str(item) for item in (value or [])] else: - out[name] = value - if "vector" not in self.schema.model_fields: - out[_DUMMY_VECTOR_FIELD] = [0.0] + out[field.name] = value + if not self.index_schema.vector_fields: + out[_DUMMY_VECTOR_FIELD] = [0.0] * _DUMMY_VECTOR_DIMENSION + self._validate_raw_record(out) return out + def _validate_raw_record(self, row: dict[str, Any]) -> None: + for field in self.index_schema.fields: + storage_name = ( + _datetime_storage_field(field.name) + if field.kind is IndexFieldKind.DATETIME + else field.name + ) + value = row.get(storage_name) + if value is None: + if not field.nullable: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} cannot be null" + ) + continue + if field.kind is IndexFieldKind.STRING: + size = len(str(value).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} is {size} UTF-8 bytes; " + f"Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.STRING_ARRAY: + if field.max_capacity is not None and len(value) > field.max_capacity: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has {len(value)} items; " + f"Milvus limit is {field.max_capacity}" + ) + for position, item in enumerate(value): + size = len(str(item).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name}[{position}] is {size} " + f"UTF-8 bytes; Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + self._validate_vector(field, value) + + def _validate_vector(self, field: IndexField, value: Sequence[float]) -> None: + if len(value) != field.dimension: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has dimension {len(value)}; " + f"expected {field.dimension}" + ) + def _model_from_milvus(self, row: dict[str, Any]) -> T: - shaped = self._restore_row(row, include_distance=False) - return self.schema.model_validate(shaped) + return self.schema.model_validate(self._restore_row(row)) def _candidate_row_from_search( self, row: dict[str, Any], *, normalize_cosine: bool = False ) -> dict[str, Any]: - shaped = self._restore_row(row.get("entity", {}), include_distance=False) + shaped = self._restore_row(row.get("entity", {})) raw_distance = row.get("distance") shaped["_distance"] = ( _cosine_distance_from_milvus(raw_distance) @@ -534,49 +617,53 @@ def _candidate_row_from_search( ) return shaped - def _restore_row( - self, row: dict[str, Any], *, include_distance: bool - ) -> dict[str, Any]: + def _restore_row(self, row: dict[str, Any]) -> dict[str, Any]: out: dict[str, Any] = {} - for name, field in self.schema.model_fields.items(): - if name == "subject_vector": - continue - if _is_datetime(field.annotation): - storage_name = _datetime_storage_field(name) - value = row.get(storage_name) - if value is None: - out[name] = None - else: - out[name] = from_timestamp(int(value)) - elif name in row: - value = row[name] + for field in self.index_schema.fields: + if field.kind is IndexFieldKind.DATETIME: + value = row.get(_datetime_storage_field(field.name)) + out[field.name] = None if value is None else from_timestamp(int(value)) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + if field.name in row: + out[field.name] = ( + row[field.name] + if row.get(_present_field(field.name), True) + else None + ) + elif field.name in row: + value = row[field.name] if ( - name in self.schema.BM25_FIELDS + field.name in self.index_schema.bm25_fields and value == "" - and _is_optional(field.annotation) + and field.nullable ): - out[name] = None - else: - out[name] = value - if include_distance and "distance" in row: - out["_distance"] = row["distance"] + value = None + out[field.name] = value return out def _write_field_value( self, row: dict[str, Any], field_name: str, value: Any ) -> None: - field = self.schema.model_fields.get(field_name) - if field is None: - row[field_name] = value - elif _is_datetime(field.annotation): + field = self.index_schema.field(field_name) + if field.kind is IndexFieldKind.DATETIME: row[_datetime_storage_field(field_name)] = ( _datetime_to_ms(value) if value is not None else None ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + row[field_name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + row[_present_field(field_name)] = present else: row[field_name] = value async def _query_raw( - self, where: Any, *, limit: int, include_vectors: bool + self, + where: Predicate | None, + *, + limit: int, + include_vectors: bool, ) -> list[dict[str, Any]]: await self.ensure_collection() client = await get_client() @@ -589,49 +676,31 @@ async def _query_raw( ) async def _query_candidate_rows( - self, where: Any, *, limit: int + self, where: Predicate | None, *, limit: int ) -> list[dict[str, Any]]: rows = await self._query_raw(where, limit=limit, include_vectors=False) - return [self._restore_row(row, include_distance=False) for row in rows] - - def _expr(self, where: Any) -> str: - if where is None: - return "" - milvus_expr = getattr(where, "milvus", None) - if isinstance(milvus_expr, str): - return milvus_expr - if not isinstance(where, str): - return str(where) - return _lancedb_expr_to_milvus(where) - - -def _lancedb_expr_to_milvus(expr: str) -> str: - """Best-effort conversion for internal EverOS LanceDB predicates.""" - converted = expr - converted = re.sub(r"\barray_has\s*\(", "array_contains(", converted) - converted = re.sub(r"\bIS\s+NULL\b", "is null", converted, flags=re.IGNORECASE) - converted = re.sub( - r"\b([A-Za-z_][A-Za-z0-9_]*)\s+IN\s+\(([^)]*)\)", - lambda m: f"{m.group(1)} in [{m.group(2)}]", - converted, - flags=re.IGNORECASE, - ) - converted = re.sub(r"(?=])=(?!=)", "==", converted) - converted = re.sub(r"\btimestamp\b", "timestamp_ms", converted) - converted = re.sub(r"\bcreated_at\b", "created_at_ms", converted) - converted = re.sub(r"\bupdated_at\b", "updated_at_ms", converted) - converted = re.sub( - r"TIMESTAMP\s+'([^']*)'", - lambda m: str(_iso_to_ms(m.group(1))), - converted, - ) - return converted + return [self._restore_row(row) for row in rows] + + def _expr(self, where: Predicate | None) -> str: + if where is not None and not isinstance(where, Predicate): + raise TypeError( + "Milvus repository predicates must use the neutral Predicate AST, " + f"got {type(where).__name__}" + ) + return render_predicate( + where, + datetime_fields=self.index_schema.datetime_fields, + ) def _datetime_storage_field(name: str) -> str: return f"{name}_ms" +def _present_field(name: str) -> str: + return f"{name}{_PRESENT_SUFFIX}" + + def _sparse_field(name: str) -> str: return f"{name}{_SPARSE_SUFFIX}" @@ -646,46 +715,6 @@ def _datetime_to_ms(value: Any) -> int: raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") -def _iso_to_ms(value: str) -> int: - aware = ensure_utc(from_iso_format(value)) - assert aware is not None - return to_timestamp_ms(aware) - - -def _is_optional(annotation: Any) -> bool: - args = get_args(annotation) - return type(None) in args - - -def _non_none_args(annotation: Any) -> tuple[Any, ...]: - return tuple(arg for arg in get_args(annotation) if arg is not type(None)) - - -def _is_datetime(annotation: Any) -> bool: - if annotation is dt.datetime: - return True - return any(arg is dt.datetime for arg in _non_none_args(annotation)) - - -def _is_list(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin is list: - return True - return any(get_origin(arg) is list for arg in _non_none_args(annotation)) - - -def _is_float(annotation: Any) -> bool: - if annotation is float: - return True - return any(arg is float for arg in _non_none_args(annotation)) - - -def _is_int(annotation: Any) -> bool: - if annotation is int: - return True - return any(arg is int for arg in _non_none_args(annotation)) - - def _sort_value(value: Any) -> Any: if value is None: fallback = ensure_utc(dt.datetime.min) @@ -695,64 +724,25 @@ def _sort_value(value: Any) -> Any: def _bm25_score_from_distance(distance: Any) -> float: - """Convert the Milvus BM25 search distance into a higher-is-better score.""" - if distance is None: - return 0.0 - return abs(float(distance)) + """Milvus BM25 is higher-is-better; expose a non-negative score.""" + return 0.0 if distance is None else max(0.0, float(distance)) def _cosine_distance_from_milvus(distance: Any) -> float | None: - """Normalize Milvus COSINE results to LanceDB-style distance. - - Milvus server / Zilliz Cloud return cosine similarity through the - ``distance`` field. Milvus Lite 3.0 reports cosine distance instead; keep - this version-gated workaround narrow because the upstream issue is expected - to be fixed after 3.0: - https://github.com/milvus-io/milvus-lite/issues/343 - - Recaller code expects LanceDB's ``1 - similarity`` distance contract. - """ + """Convert Milvus Server / Zilliz similarity to Lance-style distance.""" if distance is None: return None - value = float(distance) - if not _uses_milvus_lite_3_0_cosine_distance_bug(): - value = 1.0 - value - if value < 0.0: - return 0.0 - if value > 1.0: - return 1.0 - return value - - -def _uses_milvus_lite() -> bool: - uri = load_settings().milvus.uri - if not uri: - return True - return "://" not in uri - - -def _uses_milvus_lite_3_0_cosine_distance_bug() -> bool: - if not _uses_milvus_lite(): - return False - return _milvus_lite_version() in {"3.0", "3.0.0"} - - -def _milvus_lite_version() -> str | None: - try: - return version("milvus-lite") - except PackageNotFoundError: - return None + return min(1.0, max(0.0, 1.0 - float(distance))) def _first_result_set(results: Any) -> list[dict[str, Any]]: if not results: return [] - first = results[0] - return list(first or []) + return list(results[0] or []) async def _run(func: Any, /, *args: Any, **kwargs: Any) -> Any: return await asyncio.to_thread(func, *args, **kwargs) -__all__ = ["MilvusRepoBase"] +__all__ = ["MilvusRepoBase", "MilvusValueLimitError"] diff --git a/src/everos/infra/persistence/predicate.py b/src/everos/infra/persistence/predicate.py new file mode 100644 index 000000000..def7ca69f --- /dev/null +++ b/src/everos/infra/persistence/predicate.py @@ -0,0 +1,138 @@ +"""Backend-neutral predicate tree shared by derived-index adapters.""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from typing import Literal + +type Scalar = str | int | float | bool | dt.datetime +type ComparisonOperator = Literal["eq", "ne", "gt", "gte", "lt", "lte"] + + +class Predicate: + """Marker base class for derived-index predicates.""" + + +@dataclass(frozen=True) +class Comparison(Predicate): + field: str + operator: ComparisonOperator + value: Scalar + + +@dataclass(frozen=True) +class In(Predicate): + field: str + values: tuple[Scalar, ...] + + +@dataclass(frozen=True) +class Contains(Predicate): + field: str + value: str + + +@dataclass(frozen=True) +class IsNull(Predicate): + field: str + + +@dataclass(frozen=True) +class All(Predicate): + children: tuple[Predicate, ...] + + +@dataclass(frozen=True) +class AnyOf(Predicate): + children: tuple[Predicate, ...] + + +def compare(field: str, operator: ComparisonOperator, value: Scalar) -> Predicate: + return Comparison(field, operator, value) + + +def eq(field: str, value: Scalar) -> Predicate: + return compare(field, "eq", value) + + +def ne(field: str, value: Scalar) -> Predicate: + return compare(field, "ne", value) + + +def gt(field: str, value: Scalar) -> Predicate: + return compare(field, "gt", value) + + +def gte(field: str, value: Scalar) -> Predicate: + return compare(field, "gte", value) + + +def lt(field: str, value: Scalar) -> Predicate: + return compare(field, "lt", value) + + +def lte(field: str, value: Scalar) -> Predicate: + return compare(field, "lte", value) + + +def one_of(field: str, values: list[Scalar] | tuple[Scalar, ...]) -> Predicate: + if not values: + raise ValueError("one_of requires at least one value") + return In(field, tuple(values)) + + +def contains(field: str, value: str) -> Predicate: + return Contains(field, value) + + +def is_null(field: str) -> Predicate: + return IsNull(field) + + +def all_of(*predicates: Predicate | None) -> Predicate: + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, All): + children.extend(predicate.children) + else: + children.append(predicate) + return All(tuple(children)) + + +def any_of(*predicates: Predicate | None) -> Predicate: + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, AnyOf): + children.extend(predicate.children) + else: + children.append(predicate) + return AnyOf(tuple(children)) + + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/memory/cascade/_backfill.py b/src/everos/memory/cascade/_backfill.py index 767ee6d24..d78bd27ba 100644 --- a/src/everos/memory/cascade/_backfill.py +++ b/src/everos/memory/cascade/_backfill.py @@ -46,8 +46,7 @@ from everos.infra.ome.config import OMEConfig from everos.infra.ome.engine import OfflineEngine from everos.infra.ome.exceptions import EngineLockHeldError -from everos.infra.persistence.lancedb import ( - BUSINESS_SCHEMAS_WITH_VECTOR, +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, @@ -58,10 +57,11 @@ agent_skill_repo, atomic_fact_repo, episode_repo, + eq, foresight_repo, - get_table, knowledge_topic_repo, ) +from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR, get_table from everos.infra.persistence.markdown import AgentSkillFrontmatter from everos.infra.persistence.sqlite import cluster_repo, get_engine from everos.memory.cascade.worker import ( @@ -697,7 +697,7 @@ async def _backfill_table( and row.id not in subject_vectors ) try: - await backlog.spec.repo.update(updates, where=f"id = '{_q(row.id)}'") + await backlog.spec.repo.update(updates, where=eq("id", row.id)) except Exception: result.rows_failed += 1 logger.warning( diff --git a/src/everos/memory/cascade/handlers/_daily_log_base.py b/src/everos/memory/cascade/handlers/_daily_log_base.py index 8791a418f..52f67b781 100644 --- a/src/everos/memory/cascade/handlers/_daily_log_base.py +++ b/src/everos/memory/cascade/handlers/_daily_log_base.py @@ -30,6 +30,7 @@ from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, StructuredEntry +from everos.infra.persistence.index import all_of, eq, one_of from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 @@ -108,7 +109,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: ] existing = await self.lance_repo.find_where( - f"md_path = '{_q(md_path)}'", + eq("md_path", md_path), limit=10_000, ) owner_id, owner_type = resolve_owner(parsed.frontmatter, md_path) @@ -198,9 +199,11 @@ async def _apply_lance_changes( if to_upsert: await self.lance_repo.upsert(to_upsert) if to_delete_ids: - in_list = ", ".join(f"'{eid}'" for eid in to_delete_ids) await self.lance_repo.delete( - f"md_path = '{_q(md_path)}' AND entry_id IN ({in_list})" + all_of( + eq("md_path", md_path), + one_of("entry_id", to_delete_ids), + ) ) async def handle_deleted(self, md_path: str) -> HandlerOutcome: @@ -252,11 +255,11 @@ async def _mark_deprecated( entry may have been deleted or not yet indexed. """ app_id, project_id = scope - predicate = ( - f"owner_id = '{_q(owner_id)}' " - f"AND entry_id = '{_q(entry_id)}' " - f"AND app_id = '{_q(app_id)}' " - f"AND project_id = '{_q(project_id)}'" + predicate = all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), ) try: await self.lance_repo.update( @@ -290,8 +293,3 @@ async def _build_row( ``"default"`` so white-box callers exercising only the field mapping can omit them. """ - - -def _q(text: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return text.replace("'", "''") diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 25315d35b..1e492eec1 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -42,7 +42,13 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader -from everos.infra.persistence.index import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import ( + AgentSkill, + agent_skill_repo, + all_of, + eq, + ne, +) from everos.infra.persistence.markdown import AgentSkillFrontmatter from ..types import HandlerOutcome @@ -136,7 +142,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: # cascade's contract — skip the lookup. deleted = 0 if prior is None: - orphan_clause = f"md_path = '{_q(md_path)}' AND id != '{_q(skill_id)}'" + orphan_clause = all_of(eq("md_path", md_path), ne("id", skill_id)) orphans = await agent_skill_repo.find_where(orphan_clause, limit=1000) deleted = len(orphans) if deleted: @@ -224,8 +230,3 @@ def _join_body_and_references(body: str, references: str) -> str: if not body: return references return f"{body}\n\n{references}" - - -def _q(value: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return value.replace("'", "''") diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 46b1e6305..cd23dcc67 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -15,8 +15,9 @@ from __future__ import annotations +from everos.infra.persistence.index.predicate import Predicate from everos.memory.search import FilterNode, compile_filters -from everos.memory.search.filters import BackendFilters, compile_filters_for_backends +from everos.memory.search.filters import compile_filters_for_backends def compile_filters_for_get( @@ -26,7 +27,7 @@ def compile_filters_for_get( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: +) -> Predicate: """Compile ``/get`` filters via the shared ``compile_filters`` path. Kept as a named wrapper so ``memory.get`` consumers depend on a @@ -48,7 +49,7 @@ def compile_filters_for_get_backends( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> BackendFilters: +) -> Predicate: """Compile ``/get`` filters for every derived index backend.""" return compile_filters_for_backends( filters, diff --git a/src/everos/memory/reflection/orchestrator.py b/src/everos/memory/reflection/orchestrator.py index 547adc643..9c9e8c54d 100644 --- a/src/everos/memory/reflection/orchestrator.py +++ b/src/everos/memory/reflection/orchestrator.py @@ -32,6 +32,7 @@ from everos.core.observability.tracing import memory_span from everos.core.persistence import MemoryRoot from everos.infra.ome.context import StrategyContext +from everos.infra.persistence.index import all_of, eq, is_null from everos.memory._partition_locks import get_partition_lock from everos.memory.events import EpisodeExtracted @@ -41,21 +42,6 @@ _WAIT_TIMEOUT_SECONDS = 120.0 -def _escape_sql(value: str) -> str: - """Escape single quotes for LanceDB SQL-like ``where`` predicates. - - LanceDB has no parameterised query API; doubling the quote - (``'`` -> ``''``) is the SQL-standard escape. - - Args: - value: Raw string to escape. - - Returns: - Escaped string safe for interpolation into a WHERE clause. - """ - return value.replace("'", "''") - - class ReflectionOrchestrator: """Run one Reflection cycle for a single owner scope. @@ -531,12 +517,13 @@ async def _detect_orphans( app_id: Application scope. project_id: Project scope. """ - where = ( - f"parent_type = 'cluster' AND parent_id = '{_escape_sql(cluster_id)}' " - f"AND deprecated_by IS NULL " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where = all_of( + eq("parent_type", "cluster"), + eq("parent_id", cluster_id), + is_null("deprecated_by"), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ) orphans = await self._episode_store.find_where(where, limit=10) if orphans: @@ -814,14 +801,14 @@ async def _apply_deprecation_writes( to_deprecate=to_deprecate, merged_entry_id=merged_entry_id, ) - deprecated_ep_count = await self._deprecate_lance_episodes( + deprecated_ep_count = await self._deprecate_index_episodes( entry_ids=to_deprecate, owner_id=owner_id, app_id=app_id, project_id=project_id, merged_entry_id=merged_entry_id, ) - deprecated_fact_count = await self._deprecate_lance_facts( + deprecated_fact_count = await self._deprecate_index_facts( parent_ids=to_deprecate, owner_id=owner_id, merged_entry_id=merged_entry_id, @@ -848,7 +835,7 @@ async def _resolve_deprecation_targets( original_ids = {mid for mid, _ in original_members} return original_ids & current_ids - async def _deprecate_lance_episodes( + async def _deprecate_index_episodes( self, *, entry_ids: set[str], @@ -857,19 +844,19 @@ async def _deprecate_lance_episodes( project_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated episodes in LanceDB by entry_id. + """Mark deprecated episodes in the active derived index by entry_id. Returns: - Number of LanceDB update calls issued. + Number of derived-index update calls issued. """ coros: list[Any] = [ self._episode_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"entry_id = '{_escape_sql(eid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where=all_of( + eq("entry_id", eid), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ), ) for eid in entry_ids @@ -878,14 +865,14 @@ async def _deprecate_lance_episodes( await asyncio.gather(*coros) return len(coros) - async def _deprecate_lance_facts( + async def _deprecate_index_facts( self, *, parent_ids: set[str], owner_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated atomic facts in LanceDB. + """Mark deprecated atomic facts in the active derived index. Args: parent_ids: Parent IDs (memcell or episode) whose facts to deprecate. @@ -893,7 +880,7 @@ async def _deprecate_lance_facts( merged_entry_id: Entry ID of the replacement merged episode. Returns: - Total number of LanceDB update calls issued. + Total number of derived-index update calls issued. """ if not parent_ids: return 0 @@ -901,10 +888,10 @@ async def _deprecate_lance_facts( coros = [ self._atomic_fact_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"parent_id = '{_escape_sql(pid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND deprecated_by IS NULL" + where=all_of( + eq("parent_id", pid), + eq("owner_id", owner_id), + is_null("deprecated_by"), ), ) for pid in parent_ids diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 0f417afc0..74ceeb2d5 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,86 +1,37 @@ -"""Filters DSL → derived-index filter compiler. - -The Filters DSL is intentionally permissive at the JSON layer (so callers -can pass whatever they like and get a clean 400 if it is not supported) -and rigid at compile time. Field names are validated against a small -allow-list; operators against a closed enum; string literals are -single-quote-escaped. Timestamps are accepted as epoch milliseconds and -rendered as DataFusion ``TIMESTAMP ''`` literals. - -``owner_id`` and ``owner_type`` are the hard partition keys; they are -not part of the DSL at all. :func:`compile_filters` injects them at the -top of the compiled string from :class:`SearchRequest` and rejects any -attempt to override them inside ``filters``. - -Public surface --------------- - -The compiler exposes three primitives so adjacent subpackages -(notably ``memory.get``) can build narrower DSLs without forking the -field allow-list: - -* :data:`ALLOWED_FIELDS` — mapping ``field_name → _FieldSpec`` (column + - kind). Iterate / membership-test only; do not mutate. -* :data:`RESERVED_FIELDS` — names rejected inside any ``filters`` block. -* :func:`compile_predicate` — render one ``{field: value}`` clause to - a backend-specific predicate. Operator-map and equality-shorthand are - both handled. - -The high-level :func:`compile_filters` remains the entry point for -``/search`` (combinator-aware). +"""Validate the public filter DSL and build a backend-neutral predicate. + +Storage-specific syntax is intentionally absent from this module. LanceDB and +Milvus render the resulting predicate in their own persistence packages. """ from __future__ import annotations -import dataclasses -import datetime as _dt -import json -from typing import Any, Final - -from everos.component.utils.datetime import ( - ensure_utc, - from_iso_format, - from_timestamp, - to_iso_format, - to_timestamp_ms, -) +import datetime as dt +from dataclasses import dataclass +from typing import Any, Final, Literal + +from everos.component.utils.datetime import ensure_utc, from_iso_format, from_timestamp from everos.core.errors import FilterError as FilterError +from everos.infra.persistence.index.predicate import ( + Predicate, + all_of, + any_of, + compare, + contains, + eq, + is_null, + one_of, +) from .dto import FilterNode -# ── Allow-lists ────────────────────────────────────────────────────────── - -_OP_MAP: Final[dict[str, str]] = { - "eq": "=", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", - "in": "IN", -} -_MILVUS_OP_MAP: Final[dict[str, str]] = { - "eq": "==", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", - "in": "in", -} - -# Field kinds: ``str`` rendered as ``''``; ``ts`` rendered as -# ``TIMESTAMP ''`` (DataFusion timestamp literal); ``array_str`` -# uses DataFusion's ``array_has`` on a list column. -_FieldKind = str # one of: "str" | "ts" | "array_str" +_FieldKind = Literal["str", "ts", "array_str"] +@dataclass(frozen=True) class _FieldSpec: - __slots__ = ("column", "kind") - - def __init__(self, column: str, kind: _FieldKind) -> None: - self.column = column - self.kind = kind + column: str + kind: _FieldKind ALLOWED_FIELDS: Final[dict[str, _FieldSpec]] = { @@ -91,29 +42,11 @@ def __init__(self, column: str, kind: _FieldKind) -> None: "sender_id": _FieldSpec("sender_ids", "array_str"), } -# Fields the caller is explicitly **not** allowed to place inside -# ``filters``; they live at the top of :class:`SearchRequest` and are -# injected by :func:`compile_filters`. Rejecting them here turns a -# silent override into a 400. RESERVED_FIELDS: Final[frozenset[str]] = frozenset( {"owner_id", "owner_type", "app_id", "project_id"} ) -@dataclasses.dataclass(frozen=True) -class BackendFilters: - """Compiled filters for every supported derived index backend.""" - - lancedb: str - milvus: str - - def __str__(self) -> str: - return self.lancedb - - -# ── Public API ─────────────────────────────────────────────────────────── - - def compile_filters( node: FilterNode | None, *, @@ -121,24 +54,21 @@ def compile_filters( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: - """Compile a request's filters into a single LanceDB ``where`` string. - - The base clause always pins the hard partition keys (``owner_id`` / - ``owner_type`` and the ``app_id`` / ``project_id`` scope segments) to - the request's top-level values; anything in ``node`` is appended with - an ``AND``. Pinning app/project here is what isolates one space's rows - from another — omitting it would let a query bleed across spaces. Both - ``/search`` and ``/get`` share this compile path. - """ - return _compile_filters_backend( - node, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - backend="lancedb", - ) +) -> Predicate: + """Validate request filters and return one normalized predicate tree.""" + base: list[Predicate] = [ + eq("owner_id", owner_id), + eq("owner_type", owner_type), + eq("app_id", app_id), + eq("project_id", project_id), + ] + if owner_type == "user": + base.append(is_null("deprecated_by")) + if node is not None: + compiled = _compile_node(node.model_dump(exclude_none=True)) + if compiled is not None: + base.append(compiled) + return all_of(*base) def compile_filters_for_backends( @@ -148,74 +78,29 @@ def compile_filters_for_backends( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> BackendFilters: - """Compile request filters for every supported derived index backend.""" - return BackendFilters( - lancedb=_compile_filters_backend( - node, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - backend="lancedb", - ), - milvus=_compile_filters_backend( - node, - owner_id=owner_id, - owner_type=owner_type, - app_id=app_id, - project_id=project_id, - backend="milvus", - ), - ) - - -def _compile_filters_backend( - node: FilterNode | None, - *, - owner_id: str, - owner_type: str, - app_id: str, - project_id: str, - backend: str, -) -> str: - base = _base_clauses( +) -> Predicate: + """Compatibility name; one predicate is rendered by the active backend.""" + return compile_filters( + node, owner_id=owner_id, owner_type=owner_type, app_id=app_id, project_id=project_id, - backend=backend, ) - # Only episode / atomic_fact tables carry the ``deprecated_by`` column - # (Reflection V1 marks superseded entries). Agent tables don't have it. - if owner_type == "user": - base.append( - "deprecated_by IS NULL" if backend == "lancedb" else "deprecated_by is null" - ) - if node is None: - return " AND ".join(base) - compiled = _compile_node(node.model_dump(exclude_none=True), backend=backend) - if not compiled: - return " AND ".join(base) - return " AND ".join([*base, compiled]) - -# ── Internals ──────────────────────────────────────────────────────────── - -def _compile_node(raw: dict[str, Any], *, backend: str = "lancedb") -> str: - """Walk one DSL node; return the matching SQL fragment (no leading parens). - - Empty nodes yield ``""`` so :func:`compile_filters` can skip the - trailing ``AND``. - """ - raw = dict(raw) # never mutate the caller's dict - parts: list[str] = [] +def _compile_node(raw: dict[str, Any]) -> Predicate | None: + raw = dict(raw) + parts: list[Predicate] = [] if (and_list := raw.pop("AND", None)) is not None: - parts.append(_compile_combinator(and_list, "AND", backend=backend)) + combinator = _compile_combinator(and_list, "AND") + if combinator is not None: + parts.append(combinator) if (or_list := raw.pop("OR", None)) is not None: - parts.append(_compile_combinator(or_list, "OR", backend=backend)) + combinator = _compile_combinator(or_list, "OR") + if combinator is not None: + parts.append(combinator) for field, value in raw.items(): if field in RESERVED_FIELDS: @@ -224,195 +109,101 @@ def _compile_node(raw: dict[str, Any], *, backend: str = "lancedb") -> str: ) if field not in ALLOWED_FIELDS: raise FilterError(f"unsupported filter field: {field!r}") - parts.append(compile_predicate(field, value, backend=backend)) + parts.append(compile_predicate(field, value)) - # Drop empty fragments coming from empty AND/OR arrays. - parts = [p for p in parts if p] - if not parts: - return "" - if len(parts) == 1: - return parts[0] - return " AND ".join(parts) + return all_of(*parts) if parts else None def _compile_combinator( - children: list[dict[str, Any]], op: str, *, backend: str = "lancedb" -) -> str: - """Render an ``AND`` / ``OR`` array of child nodes.""" + children: list[dict[str, Any]], op: Literal["AND", "OR"] +) -> Predicate | None: if not isinstance(children, list): raise FilterError(f"{op} expects an array of nodes") - fragments: list[str] = [] + fragments: list[Predicate] = [] for child in children: if not isinstance(child, dict): raise FilterError(f"{op} children must be objects") - compiled = _compile_node(child, backend=backend) - if compiled: - fragments.append(f"({compiled})") + compiled = _compile_node(child) + if compiled is not None: + fragments.append(compiled) if not fragments: - return "" - if len(fragments) == 1: - # No need for the surrounding combinator when only one effective child. - return fragments[0] - glue = f" {op} " - return "(" + glue.join(fragments) + ")" - - -def compile_predicate(field: str, value: Any, *, backend: str = "lancedb") -> str: - """Render one ``"": `` clause to SQL. - - Public primitive — :mod:`memory.get` builds a flat (no AND/OR) - DSL on top of it. Callers must pre-validate ``field`` against - :data:`ALLOWED_FIELDS` and :data:`RESERVED_FIELDS`; this function - will ``KeyError`` on unknown fields. + return None + return all_of(*fragments) if op == "AND" else any_of(*fragments) - ``value`` is either a scalar (equality shorthand) or an - ``{"": }`` map. Mixing multiple operators in one - dict is allowed and folds with ``AND``:: - "timestamp": {"gte": 1, "lt": 2} - → (timestamp >= TIMESTAMP '...' AND timestamp < TIMESTAMP '...') - """ +def compile_predicate(field: str, value: Any) -> Predicate: + """Validate and normalize one field clause into the neutral AST.""" spec = ALLOWED_FIELDS[field] if isinstance(value, dict): if not value: raise FilterError(f"empty operator map for field {field!r}") - clauses = [ - _compile_op_clause(spec, field, op, op_val, backend=backend) - for op, op_val in value.items() - ] - if len(clauses) == 1: - return clauses[0] - return "(" + " AND ".join(clauses) + ")" - # Equality shorthand. - return _compile_op_clause(spec, field, "eq", value, backend=backend) - - -def _compile_op_clause( - spec: _FieldSpec, field: str, op: str, value: Any, *, backend: str = "lancedb" -) -> str: - """Render a single `` `` clause.""" - if op not in _OP_MAP: + return all_of( + *( + _compile_op_clause(spec, field, op, op_value) + for op, op_value in value.items() + ) + ) + return _compile_op_clause(spec, field, "eq", value) + + +def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> Predicate: + if op not in {"eq", "ne", "gt", "gte", "lt", "lte", "in"}: raise FilterError(f"unsupported operator {op!r} on field {field!r}") - op_map = _MILVUS_OP_MAP if backend == "milvus" else _OP_MAP - sql_op = op_map[op] - column = _column_for_backend(spec, backend) if spec.kind == "array_str": - # Only equality / membership make sense on a list column. - fn = "array_contains" if backend == "milvus" else "array_has" if op == "eq": - literal = _render_str_literal(_require_str(value, field), backend) - return f"{fn}({column}, {literal})" + return contains(spec.column, _require_str(value, field)) if op == "in": - items = _require_list(value, field) - literals = [ - _render_str_literal(_require_str(v, field), backend) for v in items - ] - inner = " OR ".join(f"{fn}({column}, {lit})" for lit in literals) - return f"({inner})" + values = _require_list(value, field) + return any_of( + *(contains(spec.column, _require_str(item, field)) for item in values) + ) raise FilterError(f"operator {op!r} is not supported on array field {field!r}") if op == "in": - items = _require_list(value, field) - literals = [ - _render_literal(v, spec.kind, field, backend=backend) for v in items - ] - if backend == "milvus": - return f"{column} in [{', '.join(literals)}]" - return f"{column} IN ({', '.join(literals)})" - - literal = _render_literal(value, spec.kind, field, backend=backend) - return f"{column} {sql_op} {literal}" - - -# ── Literal rendering ──────────────────────────────────────────────────── + values = _require_list(value, field) + return one_of( + spec.column, + [_normalize_literal(item, spec.kind, field) for item in values], + ) + return compare( + spec.column, + op, # type: ignore[arg-type] + _normalize_literal(value, spec.kind, field), + ) -def _render_literal( - value: Any, kind: _FieldKind, field: str, *, backend: str = "lancedb" -) -> str: +def _normalize_literal(value: Any, kind: _FieldKind, field: str): # type: ignore[no-untyped-def] if kind == "str": - return _render_str_literal(_require_str(value, field), backend) + return _require_str(value, field) if kind == "ts": - if backend == "milvus": - return str(_render_ts_ms(value, field)) - return f"TIMESTAMP '{_render_ts(value, field)}'" + return _normalize_timestamp(value, field) raise FilterError(f"unsupported field kind {kind!r} for field {field!r}") -def _render_ts(value: Any, field: str) -> str: - """Accept epoch ms (int / float) or an ISO 8601 string; emit ISO.""" - if isinstance(value, bool): # bools subclass int — reject early - raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - if isinstance(value, (int, float)): - return to_iso_format(from_timestamp(int(value))) - if isinstance(value, str): - # Trust the caller-supplied ISO string but escape quotes defensively. - if "'" in value: - raise FilterError(f"timestamp string for {field!r} contains a quote") - return value - if isinstance(value, _dt.datetime): - return to_iso_format(value) - raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - - -def _render_ts_ms(value: Any, field: str) -> int: +def _normalize_timestamp(value: Any, field: str) -> dt.datetime: if isinstance(value, bool): raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - if isinstance(value, (int, float)): - return int(value) - if isinstance(value, str): - if "'" in value: - raise FilterError(f"timestamp string for {field!r} contains a quote") - try: - aware = ensure_utc(from_iso_format(value)) - except (TypeError, ValueError) as exc: - raise FilterError( - f"timestamp value for {field!r} must be ms or ISO string" - ) from exc - assert aware is not None - return to_timestamp_ms(aware) - if isinstance(value, _dt.datetime): - aware = ensure_utc(value) - assert aware is not None - return to_timestamp_ms(aware) + try: + if isinstance(value, (int, float)): + return from_timestamp(int(value)) + if isinstance(value, str): + if "'" in value: + raise FilterError(f"timestamp string for {field!r} contains a quote") + parsed = ensure_utc(from_iso_format(value)) + assert parsed is not None + return parsed + if isinstance(value, dt.datetime): + parsed = ensure_utc(value) + assert parsed is not None + return parsed + except (TypeError, ValueError) as exc: + raise FilterError( + f"timestamp value for {field!r} must be ms or ISO string" + ) from exc raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") -def _render_str_literal(value: str, backend: str) -> str: - if backend == "milvus": - return json.dumps(value, ensure_ascii=False) - return f"'{_escape_str(value)}'" - - -def _base_clauses( - *, - owner_id: str, - owner_type: str, - app_id: str, - project_id: str, - backend: str, -) -> list[str]: - eq = "==" if backend == "milvus" else "=" - return [ - f"owner_id {eq} {_render_str_literal(owner_id, backend)}", - f"owner_type {eq} {_render_str_literal(owner_type, backend)}", - f"app_id {eq} {_render_str_literal(app_id, backend)}", - f"project_id {eq} {_render_str_literal(project_id, backend)}", - ] - - -def _column_for_backend(spec: _FieldSpec, backend: str) -> str: - if backend == "milvus" and spec.kind == "ts": - return f"{spec.column}_ms" - return spec.column - - -def _escape_str(value: str) -> str: - """Double single quotes — SQL-standard escape for a single-quoted literal.""" - return value.replace("'", "''") - - def _require_str(value: Any, field: str) -> str: if not isinstance(value, str): raise FilterError(f"value for {field!r} must be a string") @@ -423,3 +214,12 @@ def _require_list(value: Any, field: str) -> list[Any]: if not isinstance(value, list) or not value: raise FilterError(f"value for {field!r} with 'in' must be a non-empty list") return value + + +__all__ = [ + "ALLOWED_FIELDS", + "RESERVED_FIELDS", + "compile_filters", + "compile_filters_for_backends", + "compile_predicate", +] diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index 0c1831c79..324084829 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -14,7 +14,14 @@ from everalgo.types import Candidate -from everos.infra.persistence.index import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import ( + AgentSkill, + Predicate, + agent_skill_repo, + all_of, + any_of, + contains, +) from .base import ( RecallerDeps, @@ -23,10 +30,6 @@ ) -def _q(value: str) -> str: - return value.replace("'", "''") - - class AgentSkillRecaller: """BM25 + vector recall over the LanceDB ``agent_skill`` table.""" @@ -38,7 +41,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -61,7 +64,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] @@ -76,7 +79,7 @@ async def dense_recall( ] async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Predicate, *, limit: int ) -> list[Candidate]: """Skills whose ``source_case_ids`` intersect ``case_ids``. Filter is ``array_has`` OR-ed per id (same as @@ -88,7 +91,9 @@ async def fetch_by_case_ids( """ if not case_ids: return [] - clause = " OR ".join(f"array_has(source_case_ids, '{_q(c)}')" for c in case_ids) - full_where = f"({where}) AND ({clause})" + full_where = all_of( + where, + any_of(*(contains("source_case_ids", case_id) for case_id in case_ids)), + ) rows = await agent_skill_repo.search(where=full_where, limit=limit) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index f78b3f50a..885457e45 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -21,7 +21,13 @@ from everalgo.types import Candidate, FactCandidate -from everos.infra.persistence.index import AtomicFact, atomic_fact_repo +from everos.infra.persistence.index import ( + AtomicFact, + Predicate, + all_of, + atomic_fact_repo, + one_of, +) from .base import ( RecallerDeps, @@ -46,7 +52,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery (see EpisodeRecaller docstring).""" terms = [term for term in self._deps.tokenizer.tokenize(query) if term] @@ -64,7 +70,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Cosine ANN recall over the atomic_fact table. @@ -91,7 +97,7 @@ async def dense_recall( async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None = None, @@ -152,15 +158,13 @@ async def facts_for_episodes( async def _query_facts_for_parents( self, parent_to_eps: dict[str, list[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None, ) -> list[dict[str, Any]]: """Construct and execute the LanceDB query for parent_id IN (...).""" - quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) - clause = f"parent_id IN ({quoted})" - full_where = f"({where}) AND ({clause})" + full_where = all_of(where, one_of("parent_id", list(parent_to_eps))) # Milvus server / Zilliz Cloud reject search topK values above 1024. # The fact expansion only needs a bounded candidate pool for top-N # competition, so keep the same cap for every derived-index backend. @@ -182,7 +186,3 @@ def _build_parent_to_episode_map( if pid: parent_to_eps[pid].append(ep_id) return parent_to_eps - - -def _q(value: str) -> str: - return value.replace("'", "''") diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index 1f2c13821..f24e8ef8a 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,7 +7,14 @@ from everalgo.types import Candidate -from everos.infra.persistence.index import Episode, active_backend, episode_repo +from everos.core.observability.logging import get_logger +from everos.infra.persistence.index import ( + Episode, + Predicate, + all_of, + episode_repo, + one_of, +) from .base import ( RecallerDeps, @@ -29,8 +36,8 @@ def _inject_parent_id(candidates: list[Candidate]) -> list[Candidate]: ] -def _q(value: str) -> str: - return value.replace("'", "''") +logger = get_logger(__name__) +_FETCH_ALL_OWNER_LIMIT = 10_000 class EpisodeRecaller: @@ -44,7 +51,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery. @@ -69,7 +76,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] @@ -84,19 +91,19 @@ async def dense_recall( ] async def sparse_recall_as_child( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Sparse recall returning episodes as MaxSim child candidates.""" return _inject_parent_id(await self.sparse_recall(query, where, limit=limit)) async def dense_recall_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Dense recall (body vector ANN) returning as MaxSim children.""" return _inject_parent_id(await self.dense_recall(vector, where, limit=limit)) async def dense_recall_subject( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """ANN over the ``subject_vector`` column. @@ -105,19 +112,11 @@ async def dense_recall_subject( """ if not vector: return [] - if active_backend() == "milvus": - return await self.dense_recall(vector, where, limit=limit) - from everos.infra.persistence.lancedb import get_table - - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("subject_vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() + rows = await episode_repo.dense_search( + vector, + where, + limit=limit, + vector_field="subject_vector", ) return [ row_to_candidate( @@ -129,23 +128,28 @@ async def dense_recall_subject( ] async def dense_recall_subject_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Subject-vector ANN returning as MaxSim children.""" candidates = await self.dense_recall_subject(vector, where, limit=limit) return _inject_parent_id(candidates) - async def fetch_all_for_owner(self, where: str) -> list[Candidate]: + async def fetch_all_for_owner(self, where: Predicate) -> list[Candidate]: """Flat scan — all episodes for this owner, keyed by entry_id. Cluster membership matching in ``acluster_retrieve`` compares ``Candidate.id`` against ``Cluster.members``. Both are now episode entry_ids regardless of parent_type. - No ``limit`` — the full owner partition is required for cluster - membership matching. + The scan is capped at :data:`_FETCH_ALL_OWNER_LIMIT` to bound memory. + Hitting the cap is logged because cluster matching may be incomplete. """ - rows = await episode_repo.search(where=where, limit=10_000) + rows = await episode_repo.search(where=where, limit=_FETCH_ALL_OWNER_LIMIT) + if len(rows) == _FETCH_ALL_OWNER_LIMIT: + logger.warning( + "episode_owner_scan_truncated", + limit=_FETCH_ALL_OWNER_LIMIT, + ) result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") @@ -163,12 +167,11 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: return result async def fetch_by_entry_ids( - self, entry_ids: list[str], where: str + self, entry_ids: list[str], where: Predicate ) -> list[Candidate]: """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" if not entry_ids: return [] - quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) - full_where = f"({where}) AND (entry_id IN ({quoted}))" + full_where = all_of(where, one_of("entry_id", entry_ids)) rows = await episode_repo.search(where=full_where, limit=len(entry_ids)) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py index 2040ace17..6ee1a4d23 100644 --- a/src/everos/service/knowledge.py +++ b/src/everos/service/knowledge.py @@ -1148,15 +1148,15 @@ def _validate_scope_id(value: str, name: str) -> None: raise ValueError(f"{name} contains invalid characters: {value!r}") -def compile_knowledge_where(app_id: str, project_id: str) -> str: - """Build a LanceDB ``where`` clause scoped to the given tenant. +def compile_knowledge_where(app_id: str, project_id: str): # type: ignore[no-untyped-def] + """Build a backend-neutral predicate scoped to the given tenant. Args: app_id: Tenant application identifier. project_id: Tenant project identifier. Returns: - SQL-style predicate string safe for use in LanceDB ``where`` parameter. + Predicate safe for either derived-index backend. Raises: ValueError: If either id contains invalid characters. @@ -1164,10 +1164,9 @@ def compile_knowledge_where(app_id: str, project_id: str) -> str: _validate_scope_id(app_id, "app_id") _validate_scope_id(project_id, "project_id") - def _esc(v: str) -> str: - return v.replace("'", "''") + from everos.infra.persistence.index import all_of, eq - return f"app_id = '{_esc(app_id)}' AND project_id = '{_esc(project_id)}'" + return all_of(eq("app_id", app_id), eq("project_id", project_id)) # ── Recall helpers ─────────────────────────────────────────────────────────── diff --git a/tests/integration/test_cascade_cli_integration.py b/tests/integration/test_cascade_cli_integration.py index 22511ddf1..c89ad7f84 100644 --- a/tests/integration/test_cascade_cli_integration.py +++ b/tests/integration/test_cascade_cli_integration.py @@ -124,7 +124,7 @@ def test_sync_with_path_outside_root_errors( # between the two tokens. output = result.stdout + (result.stderr or "") plain_output = _strip_ansi(output) - assert re.search(r"not under[^\w]+memory root", plain_output), output + assert re.search(r"not[^\w]+under[^\w]+memory[^\w]+root", plain_output), output def test_sync_with_unmatched_path( diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py new file mode 100644 index 000000000..99734acd8 --- /dev/null +++ b/tests/integration/test_milvus_remote.py @@ -0,0 +1,217 @@ +"""Cross-backend behavior against Milvus Server or Zilliz Cloud. + +Set ``EVEROS_TEST_MILVUS_URI`` and, when required, +``EVEROS_TEST_MILVUS_TOKEN``. The same test is used for self-hosted and cloud +endpoints and creates uniquely prefixed, disposable collections. +""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import os +import uuid + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +_URI = os.environ.get("EVEROS_TEST_MILVUS_URI", "") + +pytestmark = pytest.mark.skipif( + not _URI, + reason="EVEROS_TEST_MILVUS_URI is not configured", +) + + +@pytest_asyncio.fixture(autouse=True) +async def _remote_milvus(monkeypatch: pytest.MonkeyPatch): + prefix = f"everos_e2e_{uuid.uuid4().hex}" + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", _URI) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", + os.environ.get("EVEROS_TEST_MILVUS_TOKEN", ""), + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", + os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", ""), + ) + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", prefix) + load_settings.cache_clear() + + from everos.infra.persistence.index import episode_repo, startup + + try: + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + await startup() + else: + await episode_repo._repo().ensure_collection() # type: ignore[attr-defined] + yield + finally: + from everos.infra.persistence.index import drop_business_tables, shutdown + + await drop_business_tables() + await shutdown() + load_settings.cache_clear() + + +def _episode( + *, + row_id: str, + entry_id: str, + session_id: str, + text: str, + vector_axis: int, + subject_axis: int, + timestamp: dt.datetime, +): # type: ignore[no-untyped-def] + from everos.infra.persistence.index import Episode + + vector = [0.0] * 1024 + vector[vector_axis] = 1.0 + subject_vector = [0.0] * 1024 + subject_vector[subject_axis] = 1.0 + return Episode( + id=row_id, + entry_id=entry_id, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + session_id=session_id, + timestamp=timestamp, + parent_id=f"mc_{entry_id}", + sender_ids=["user"], + subject=f"subject {text}", + episode=text, + episode_tokens=text, + md_path="test_app/test_project/users/u1/episodes/day.md", + content_sha256=entry_id, + vector=vector, + subject_vector=subject_vector, + ) + + +async def test_remote_milvus_matches_derived_index_contract() -> None: + from everos.infra.persistence.index import ( + Episode, + UserProfile, + episode_repo, + eq, + user_profile_repo, + ) + from everos.memory.search import FilterNode + from everos.memory.search.filters import compile_filters + + first_vector = [1.0] + [0.0] * 1023 + first_subject = [0.0, 1.0] + [0.0] * 1022 + now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + await episode_repo.upsert( + [ + _episode( + row_id="u1_ep1", + entry_id="ep1", + session_id="abc=", + text="red apple memory", + vector_axis=0, + subject_axis=1, + timestamp=now, + ), + _episode( + row_id="u1_ep2", + entry_id="ep2", + session_id="other", + text="blue banana memory", + vector_axis=1, + subject_axis=0, + timestamp=now + dt.timedelta(seconds=1), + ), + ] + ) + + where = compile_filters( + None, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + rows = await episode_repo.find_where(where, limit=10) + assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} + assert await episode_repo.count() == 2 + + equals_filter = compile_filters( + FilterNode.model_validate({"session_id": "abc="}), + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + equals_rows = await episode_repo.find_where(equals_filter, limit=10) + assert [row.id for row in equals_rows] == ["u1_ep1"] + + sparse = await episode_repo.sparse_search( + ["apple"], where, columns=Episode.BM25_FIELDS, limit=5 + ) + assert sparse[0]["id"] == "u1_ep1" + assert sparse[0]["_score"] > 0 + + dense = await episode_repo.dense_search(first_vector, where, limit=5) + assert dense[0]["id"] == "u1_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + by_subject = await episode_repo.dense_search( + first_subject, + where, + limit=5, + vector_field="subject_vector", + ) + assert by_subject[0]["id"] == "u1_ep1" + assert by_subject[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + page, total = await episode_repo.find_where_paginated( + where, + sort_by="timestamp", + page=1, + page_size=1, + max_fetch=10, + ) + assert total == 2 + assert len(page) == 1 + + concurrent = await asyncio.gather( + *(episode_repo.find_where(where, limit=10) for _ in range(4)) + ) + assert all(len(result) == 2 for result in concurrent) + + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + profile = UserProfile( + id="u1", + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + summary="initial profile", + explicit_info_json="[]", + implicit_traits_json="[]", + profile_timestamp_ms=1, + md_path="test_app/test_project/users/u1/user.md", + content_sha256="profile-v1", + ) + await user_profile_repo.upsert([profile]) + await user_profile_repo.update( + {"summary": "updated profile"}, where=eq("id", "u1") + ) + updated_profile = await user_profile_repo.get_by_id("u1") + assert updated_profile is not None + assert updated_profile.summary == "updated profile" + + assert ( + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + == 2 + ) + assert await episode_repo.count() == 0 diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 63b91c2fd..7c7fa5362 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -19,6 +19,7 @@ import hashlib import json from pathlib import Path +from typing import Any import numpy as np import pytest @@ -34,9 +35,11 @@ ) from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter +from everos.infra.persistence.predicate import Predicate from everos.infra.persistence.sqlite import cluster_repo, reflection_report_repo from everos.memory._partition_locks import _reset_for_tests from everos.memory.reflection.orchestrator import ReflectionOrchestrator @@ -69,10 +72,24 @@ async def embed(self, text: str) -> list[float]: class _EpisodeRepo(LanceDailyLogRepoBase[LanceEpisode]): schema = LanceEpisode + async def find_where( + self, where: str | Predicate, *, limit: int = 100 + ) -> list[LanceEpisode]: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + return await super().find_where(rendered, limit=limit) + + async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + await super().update(updates, where=rendered) + class _AtomicFactRepo(LanceDailyLogRepoBase[AtomicFact]): schema = AtomicFact + async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + await super().update(updates, where=rendered) + # --------------------------------------------------------------------------- # Fixtures diff --git a/tests/integration/test_tiers/test_upgrade_path.py b/tests/integration/test_tiers/test_upgrade_path.py index 7d8c2425f..72a35bb5a 100644 --- a/tests/integration/test_tiers/test_upgrade_path.py +++ b/tests/integration/test_tiers/test_upgrade_path.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import hashlib from pathlib import Path @@ -41,6 +42,18 @@ async def _episode_rows(owner_id: str) -> list[dict]: return await table.query().where(f"owner_id = '{owner_id}'").to_list() +async def _wait_for_episode_rows( + owner_id: str, expected: int, *, deadline_seconds: float +) -> list[dict]: + """Wait for every logical entry in a debounced daily-log update.""" + async with asyncio.timeout(deadline_seconds): + while True: + rows = await _episode_rows(owner_id) + if len(rows) >= expected: + return rows + await asyncio.sleep(0.1) + + async def _atomic_fact_rows(owner_id: str) -> list[dict]: table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) return await table.query().where(f"owner_id = '{owner_id}'").to_list() @@ -118,7 +131,10 @@ async def test_tier1_to_tier2_upgrade_via_backfill( # below directly against the LanceDB table. await wait_drained(deadline_seconds=40.0) - rows = await _episode_rows("u_alice") + # The queue drains file-level work, but another append to the same + # daily log can arrive just after that drain. Wait for the logical + # entry count as the end-to-end completion condition. + rows = await _wait_for_episode_rows("u_alice", _N_ITEMS, deadline_seconds=40.0) assert len(rows) == _N_ITEMS assert all(r["vector"] is None for r in rows), ( "Tier 1 (no embed) must write every episode with vector=NULL" diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index b02d2e78b..6d3f2c59d 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -127,7 +127,6 @@ def test_index_milvus_defaults() -> None: assert s.milvus.token.get_secret_value() == "" assert s.milvus.db_name == "" assert s.milvus.consistency_level == "Session" - assert s.milvus.dimension == 1024 assert s.milvus.collection_prefix == "everos" @@ -137,14 +136,12 @@ def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("EVEROS_MILVUS__TOKEN", "secret") monkeypatch.setenv("EVEROS_MILVUS__DB_NAME", "tenant_a") monkeypatch.setenv("EVEROS_MILVUS__CONSISTENCY_LEVEL", "Strong") - monkeypatch.setenv("EVEROS_MILVUS__DIMENSION", "1536") s = Settings() assert s.index.backend == "milvus" assert s.milvus.uri == "http://localhost:19530" assert s.milvus.token.get_secret_value() == "secret" assert s.milvus.db_name == "tenant_a" assert s.milvus.consistency_level == "Strong" - assert s.milvus.dimension == 1536 def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/unit/test_core/test_persistence/test_memory_root.py b/tests/unit/test_core/test_persistence/test_memory_root.py index efca98227..6da6ccd9a 100644 --- a/tests/unit/test_core/test_persistence/test_memory_root.py +++ b/tests/unit/test_core/test_persistence/test_memory_root.py @@ -91,8 +91,6 @@ def test_dotfile_paths(tmp_path: Path) -> None: mr = MemoryRoot(tmp_path) assert mr.index_dir == tmp_path / ".index" assert mr.lancedb_dir == tmp_path / ".index" / "lancedb" - assert mr.milvus_dir == tmp_path / ".index" / "milvus" - assert mr.milvus_db == tmp_path / ".index" / "milvus" / "milvus.db" assert mr.sqlite_dir == tmp_path / ".index" / "sqlite" assert mr.system_db == tmp_path / ".index" / "sqlite" / "system.db" assert mr.lock_file == tmp_path / ".lock" @@ -106,7 +104,6 @@ def test_ensure_creates_required_dirs(tmp_path: Path) -> None: assert mr.index_dir.is_dir() assert mr.sqlite_dir.is_dir() assert mr.lancedb_dir.is_dir() - assert mr.milvus_dir.is_dir() assert mr.tmp_dir.is_dir() # User-visible dirs are NOT pre-created. assert not mr.agents_dir().exists() diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py index 8d265d81f..d6fdbeb25 100644 --- a/tests/unit/test_infra/test_milvus/test_repo.py +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -1,128 +1,148 @@ -"""Real Milvus Lite coverage for the optional derived index backend.""" +"""Unit coverage for the remote Milvus derived-index adapter.""" from __future__ import annotations import datetime as dt -import importlib.util -import uuid -from pathlib import Path import pytest -import pytest_asyncio from everos.config import load_settings - -pytestmark = pytest.mark.skipif( - importlib.util.find_spec("pymilvus") is None, - reason="pymilvus optional extra is not installed", +from everos.infra.persistence.index import Episode, episode_repo, user_profile_repo +from everos.infra.persistence.index.schema import schema_for +from everos.infra.persistence.milvus import repository +from everos.infra.persistence.milvus.milvus_manager import ( + MilvusConfigurationError, + _resolve_uri, +) +from everos.infra.persistence.milvus.repository import ( + MilvusRepoBase, + MilvusValueLimitError, ) -def test_milvus_score_normalization_handles_lite_and_cloud( - monkeypatch: pytest.MonkeyPatch, -) -> None: - from everos.infra.persistence.milvus import repository - - monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) - monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0") - load_settings.cache_clear() - assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) - assert repository._bm25_score_from_distance(-1.5) == pytest.approx(1.5) - - monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.0") - assert repository._cosine_distance_from_milvus(0.25) == pytest.approx(0.25) - - monkeypatch.setenv("EVEROS_MILVUS__URI", "https://example.zillizcloud.com") - load_settings.cache_clear() - assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) - assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) - - monkeypatch.delenv("EVEROS_MILVUS__URI", raising=False) - monkeypatch.setattr(repository, "_milvus_lite_version", lambda: "3.0.1") - load_settings.cache_clear() - assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) - - -@pytest_asyncio.fixture(autouse=True) -async def _milvus_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): - monkeypatch.setenv("EVEROS_ROOT", str(tmp_path)) +@pytest.fixture(autouse=True) +def _reset_state(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") - monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", f"test_{uuid.uuid4().hex}") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://milvus.example:19530") + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", "unit_test") load_settings.cache_clear() + MilvusRepoBase._reset_locks_for_tests() yield - from everos.infra.persistence.index import shutdown - - await shutdown() + MilvusRepoBase._reset_locks_for_tests() load_settings.cache_clear() -async def test_milvus_episode_repo_upsert_search_filter_and_delete() -> None: - from everos.infra.persistence.index import Episode, episode_repo - from everos.memory.search.filters import compile_filters_for_backends - - v_apple = [0.0] * 1024 - v_apple[0] = 1.0 - v_banana = [0.0] * 1024 - v_banana[1] = 1.0 - now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) - - await episode_repo.upsert( - [ - Episode( - id="u1_ep1", - entry_id="ep1", - owner_id="u1", - owner_type="user", - app_id="default", - project_id="default", - session_id="s1", - timestamp=now, - parent_id="mc1", - sender_ids=["user"], - episode="red apple memory", - episode_tokens="red apple memory", - md_path="default_app/default_project/users/u1/episodes/day.md", - content_sha256="a", - vector=v_apple, - ), - Episode( - id="u1_ep2", - entry_id="ep2", - owner_id="u1", - owner_type="user", - app_id="default", - project_id="default", - session_id="s1", - timestamp=now + dt.timedelta(seconds=1), - parent_id="mc2", - sender_ids=["assistant"], - episode="blue banana memory", - episode_tokens="blue banana memory", - md_path="default_app/default_project/users/u1/episodes/day.md", - content_sha256="b", - vector=v_banana, - ), - ] +def _episode(**overrides): # type: ignore[no-untyped-def] + values = { + "id": "u1_ep1", + "entry_id": "ep1", + "owner_id": "u1", + "owner_type": "user", + "session_id": "abc=", + "timestamp": dt.datetime(2026, 1, 1, tzinfo=dt.UTC), + "parent_id": "mc1", + "sender_ids": ["user"], + "episode": "red apple memory", + "episode_tokens": "red apple memory", + "md_path": "default_app/default_project/users/u1/episodes/day.md", + "content_sha256": "a", + "vector": [1.0] + [0.0] * 1023, + "subject_vector": [0.0, 1.0] + [0.0] * 1022, + } + values.update(overrides) + return Episode(**values) + + +def test_remote_uri_is_required_and_local_paths_are_rejected() -> None: + settings = load_settings().milvus.model_copy(update={"uri": ""}) + with pytest.raises(MilvusConfigurationError, match="requires"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "/tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match="remote http"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "file:///tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match=r"http\(s\)"): + _resolve_uri(settings) + + +def test_neutral_schema_tracks_every_model_field_and_dense_vector() -> None: + schema = schema_for(Episode) + assert {field.name for field in schema.fields} == set(Episode.model_fields) + assert [field.name for field in schema.vector_fields] == [ + "vector", + "subject_vector", + ] + + +def test_record_conversion_stores_every_dense_vector_with_presence() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + record = milvus_repo._to_milvus_record(_episode()) + assert len(record["vector"]) == 1024 + assert len(record["subject_vector"]) == 1024 + assert record["vector__present"] is True + assert record["subject_vector__present"] is True + + missing = milvus_repo._to_milvus_record(_episode(subject_vector=None)) + assert missing["subject_vector__present"] is False + assert missing["subject_vector"] == [0.0] * 1024 + + +def test_update_fetch_preserves_dummy_vector_for_scalar_only_tables() -> None: + milvus_repo = user_profile_repo._repo() # type: ignore[attr-defined] + assert "_everos_dummy_vector" in milvus_repo._output_fields(include_vectors=True) + assert "_everos_dummy_vector" not in milvus_repo._output_fields( + include_vectors=False ) - filters = compile_filters_for_backends(None, owner_id="u1", owner_type="user") - rows = await episode_repo.find_where(filters, limit=10) - assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} - assert rows[0].timestamp.utcoffset() == dt.timedelta(0) - assert await episode_repo.count() == 2 - sparse = await episode_repo.sparse_search( - ["apple"], filters, columns=Episode.BM25_FIELDS, limit=5 - ) - assert [row["id"] for row in sparse] == ["u1_ep1"] - assert sparse[0]["_score"] > 0 +def test_record_conversion_reports_varchar_array_and_vector_limits() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + with pytest.raises(MilvusValueLimitError, match=r"episode is .* UTF-8 bytes"): + milvus_repo._to_milvus_record(_episode(episode="x" * 65_536)) + with pytest.raises(MilvusValueLimitError, match="sender_ids has 257 items"): + milvus_repo._to_milvus_record(_episode(sender_ids=["u"] * 257)) + with pytest.raises(MilvusValueLimitError, match="dimension 2"): + milvus_repo._validate_vector( + milvus_repo.index_schema.field("vector"), + [1.0, 0.0], + ) - dense = await episode_repo.dense_search(v_apple, filters, limit=5) - assert dense[0]["id"] == "u1_ep1" - assert dense[0]["_distance"] == pytest.approx(0.0) - deleted = await episode_repo.delete_by_md_path( - "default_app/default_project/users/u1/episodes/day.md" - ) - assert deleted == 2 - assert await episode_repo.count() == 0 +def test_server_score_normalization() -> None: + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) + + +async def test_collection_metadata_is_cached_after_startup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + + class _FakeClient: + has_calls = 0 + describe_calls = 0 + + def has_collection(self, name: str) -> bool: + self.has_calls += 1 + return True + + def describe_collection(self, name: str): # type: ignore[no-untyped-def] + self.describe_calls += 1 + return { + "fields": [ + {"name": field} for field in milvus_repo._stored_field_names() + ] + } + + client = _FakeClient() + + async def _fake_get_client(): # type: ignore[no-untyped-def] + return client + + monkeypatch.setattr(repository, "get_client", _fake_get_client) + await milvus_repo.ensure_collection() + await milvus_repo.ensure_collection() + assert client.has_calls == 1 + assert client.describe_calls == 1 diff --git a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py index 76990db7e..8450e3979 100644 --- a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py +++ b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py @@ -29,6 +29,7 @@ from typing import Any +from everos.infra.persistence.predicate import Comparison, Predicate from everos.memory.cascade._backfill import ( NullBackfillPresenter, _backfill_table, @@ -59,9 +60,9 @@ class _RecordingRepo: assert on the exact ``{col: value}`` shape written back.""" def __init__(self) -> None: - self.updates: list[tuple[dict[str, Any], str]] = [] + self.updates: list[tuple[dict[str, Any], Predicate]] = [] - async def update(self, values: dict[str, Any], *, where: str) -> None: + async def update(self, values: dict[str, Any], *, where: Predicate) -> None: self.updates.append((values, where)) @@ -200,11 +201,12 @@ async def test_orthogonal_partial_states_all_recover() -> None: assert result.rows_processed == 3 assert result.rows_failed == 0 - # Recover per-id updates by parsing the where clause - # (``id = 'xxx'``) — the repo double preserves call order and shape. + # Recover per-id updates from the neutral predicate tree. updates_by_id: dict[str, dict[str, Any]] = {} for values, where in repo.updates: - row_id = where.split("'")[1] + assert isinstance(where, Comparison) + assert where.field == "id" + row_id = str(where.value) updates_by_id[row_id] = values assert set(updates_by_id) == {"both", "subj_only", "prim_only"} diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py index 9f79cc562..21722c0d9 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py @@ -24,7 +24,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import AgentCase +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import AgentCaseWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.agent_case import AgentCaseHandler @@ -61,10 +63,13 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[AgentCase] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[AgentCase]: + async def find_where( + self, where: Predicate, *, limit: int = 100 + ) -> list[AgentCase]: + rendered = render_predicate(where) prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + if rendered.startswith(prefix): + md_path = rendered[len(prefix) :].rstrip("'") return [r for r in self.rows if r.md_path == md_path] return [] @@ -75,8 +80,8 @@ async def upsert(self, rows: list[AgentCase]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: - self.deletes.append(predicate) + async def delete(self, predicate: Predicate) -> None: + self.deletes.append(render_predicate(predicate)) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py index 8ed467d45..b5caa773a 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py @@ -19,7 +19,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import AgentSkill +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import AgentSkillWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import AgentSkillHandler, HandlerDeps @@ -72,29 +74,31 @@ async def delete_by_md_path(self, md_path: str) -> int: self.deletes.append(md_path) return 1 - async def find_where(self, predicate: str, *, limit: int) -> list[AgentSkill]: + async def find_where(self, predicate: Predicate, *, limit: int) -> list[AgentSkill]: """In-memory equivalent — handles only the ``md_path = '...' AND id != '...'`` shape the handler emits.""" - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + rendered = render_predicate(predicate) + if "md_path = " in rendered and "id != " in rendered: + md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] + id_lit = rendered.split("id != '")[1].split("'", 1)[0] return [ r for r in self.rows.values() if r.md_path == md_lit and r.id != id_lit ][:limit] - raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") - - async def delete(self, predicate: str) -> None: - self.predicate_deletes.append(predicate) - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") + + async def delete(self, predicate: Predicate) -> None: + rendered = render_predicate(predicate) + self.predicate_deletes.append(rendered) + if "md_path = " in rendered and "id != " in rendered: + md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] + id_lit = rendered.split("id != '")[1].split("'", 1)[0] self.rows = { rid: row for rid, row in self.rows.items() if not (row.md_path == md_lit and row.id != id_lit) } return - raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") + raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") @pytest.fixture @@ -233,7 +237,7 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( assert list(fake_repo.rows.keys()) == ["a1_new_name"] # The sweep predicate references the *new* id with the same md_path. assert fake_repo.predicate_deletes == [ - f"md_path = '{md_path}' AND id != 'a1_new_name'" + f"((md_path = '{md_path}') AND (id != 'a1_new_name'))" ] diff --git a/tests/unit/test_memory/test_cascade/test_handler_episode.py b/tests/unit/test_memory/test_cascade/test_handler_episode.py index a8af7fe1a..e23855c5d 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_episode.py +++ b/tests/unit/test_memory/test_cascade/test_handler_episode.py @@ -19,7 +19,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import Episode +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import EpisodeWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.episode import EpisodeHandler @@ -58,11 +60,12 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[Episode] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[Episode]: + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[Episode]: # Honour only the md_path = '...' filter the handler emits. + rendered = render_predicate(where) prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + if rendered.startswith(prefix): + md_path = rendered[len(prefix) :].rstrip("'") return [r for r in self.rows if r.md_path == md_path] return [] @@ -74,8 +77,8 @@ async def upsert(self, rows: list[Episode]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: - self.deletes.append(predicate) + async def delete(self, predicate: Predicate) -> None: + self.deletes.append(render_predicate(predicate)) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) diff --git a/tests/unit/test_memory/test_get/test_filters_adapter.py b/tests/unit/test_memory/test_get/test_filters_adapter.py index 4d6c48f4d..7fe6c76b3 100644 --- a/tests/unit/test_memory/test_get/test_filters_adapter.py +++ b/tests/unit/test_memory/test_get/test_filters_adapter.py @@ -19,17 +19,25 @@ import pytest -from everos.memory.get.filters_adapter import compile_filters_for_get +from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.memory.get.filters_adapter import ( + compile_filters_for_get as compile_filter_predicate_for_get, +) from everos.memory.search import FilterError, FilterNode +def compile_filters_for_get(*args: object, **kwargs: object) -> str: + """Render the neutral predicate with the legacy LanceDB test oracle.""" + return render_predicate(compile_filter_predicate_for_get(*args, **kwargs)) # type: ignore[arg-type] + + def test_no_filters_emits_base_clause() -> None: """``filters=None`` → owner + app/project scope clauses AND-joined.""" where = compile_filters_for_get(None, owner_id="u1", owner_type="user") assert where == ( - "owner_id = 'u1' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + "((owner_id = 'u1') AND (owner_type = 'user') " + "AND (app_id = 'default') AND (project_id = 'default') " + "AND (deprecated_by IS NULL))" ) @@ -43,9 +51,9 @@ def test_owner_id_quote_is_escaped() -> None: """SQL-standard double-quote escape on ``owner_id``.""" where = compile_filters_for_get(None, owner_id="o'reilly", owner_type="user") assert where == ( - "owner_id = 'o''reilly' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + "((owner_id = 'o''reilly') AND (owner_type = 'user') " + "AND (app_id = 'default') AND (project_id = 'default') " + "AND (deprecated_by IS NULL))" ) @@ -93,7 +101,7 @@ def test_timestamp_range_renders_and_folded() -> None: assert "timestamp < TIMESTAMP '" in where # The two clauses are AND-joined inside one parenthesised group. assert "(timestamp >= TIMESTAMP" in where - assert " AND timestamp < TIMESTAMP" in where + assert ") AND (timestamp < TIMESTAMP" in where def test_sender_id_in_list_renders_array_has() -> None: @@ -125,7 +133,7 @@ def test_top_level_and_renders_grouped_clause() -> None: ) where = compile_filters_for_get(node, owner_id="u1", owner_type="user") # Base clause is always first; combinator output appended. - assert where.startswith("owner_id = 'u1' AND owner_type = 'user' AND ") + assert where.startswith("((owner_id = 'u1') AND (owner_type = 'user') AND ") assert "session_id = 'sess_a'" in where assert "parent_id = 'mc_x'" in where diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 4cc5e9bfb..ea040f63c 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -27,6 +27,7 @@ Episode, UserProfile, ) +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.get import ( GetManager, GetMemoryType, @@ -65,7 +66,7 @@ async def find_where_paginated( max_fetch: int = 20000, ) -> tuple[list[Any], int]: self.last = _CallRecord( - where=str(where), + where=render_predicate(where), sort_by=sort_by, descending=descending, page=page, diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 59e85dd2e..f2c7041de 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -4,32 +4,40 @@ import pytest +from everos.infra.persistence.lancedb.predicate import render_predicate as render_lance +from everos.infra.persistence.milvus.predicate import render_predicate as render_milvus from everos.memory.search import ( FilterError, FilterNode, - compile_filters, +) +from everos.memory.search import ( + compile_filters as compile_filter_ast, ) from everos.memory.search.filters import compile_filters_for_backends + +def compile_filters(*args, **kwargs): # type: ignore[no-untyped-def] + """Render the neutral result through LanceDB for legacy syntax assertions.""" + return render_lance(compile_filter_ast(*args, **kwargs)) + + # ── Base injection ─────────────────────────────────────────────────────── def test_no_filters_emits_base_clause() -> None: where = compile_filters(None, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" - ) + assert "owner_id = 'alice'" in where + assert "owner_type = 'user'" in where + assert "app_id = 'default'" in where + assert "project_id = 'default'" in where + assert "deprecated_by IS NULL" in where def test_no_filters_agent_omits_deprecated_by() -> None: where = compile_filters(None, owner_id="bot_42", owner_type="agent") assert "deprecated_by" not in where - assert where == ( - "owner_id = 'bot_42' AND owner_type = 'agent' " - "AND app_id = 'default' AND project_id = 'default'" - ) + assert "owner_id = 'bot_42'" in where + assert "owner_type = 'agent'" in where def test_owner_type_agent_pinned() -> None: @@ -249,10 +257,10 @@ def test_empty_and_array_skips_combinator() -> None: """Empty AND/OR arrays compile to no clauses — only the base remains.""" node = FilterNode.model_validate({"AND": []}) where = compile_filters(node, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + assert where == compile_filters( + None, + owner_id="alice", + owner_type="user", ) @@ -274,11 +282,12 @@ def test_compile_filters_omits_deprecated_by_for_agent() -> None: def test_compile_filters_for_backends_preserves_lancedb_default() -> None: filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") - assert str(filters) == filters.lancedb - assert "owner_id = 'u_a'" in filters.lancedb - assert 'owner_id == "u_a"' in filters.milvus - assert "deprecated_by IS NULL" in filters.lancedb - assert "deprecated_by is null" in filters.milvus + lancedb = render_lance(filters) + milvus = render_milvus(filters) + assert "owner_id = 'u_a'" in lancedb + assert 'owner_id == "u_a"' in milvus + assert "deprecated_by IS NULL" in lancedb + assert "deprecated_by is null" in milvus def test_compile_filters_for_milvus_timestamp_and_array() -> None: @@ -286,16 +295,20 @@ def test_compile_filters_for_milvus_timestamp_and_array() -> None: {"timestamp": {"gte": 1704067200000}, "sender_id": "u_jason"} ) filters = compile_filters_for_backends(node, owner_id="u_a", owner_type="user") - assert "timestamp_ms >= 1704067200000" in filters.milvus - assert 'array_contains(sender_ids, "u_jason")' in filters.milvus - assert "TIMESTAMP '" in filters.lancedb - assert "array_has(sender_ids, 'u_jason')" in filters.lancedb + milvus = render_milvus(filters, datetime_fields={"timestamp"}) + lancedb = render_lance(filters) + assert "timestamp_ms >= 1704067200000" in milvus + assert 'array_contains(sender_ids, "u_jason")' in milvus + assert "TIMESTAMP '" in lancedb + assert "array_has(sender_ids, 'u_jason')" in lancedb def test_compile_filters_for_milvus_escapes_string_literals() -> None: node = FilterNode.model_validate({"session_id": "ses's"}) filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") - assert 'owner_id == "al\'ice"' in filters.milvus - assert 'session_id == "ses\'s"' in filters.milvus - assert "owner_id = 'al''ice'" in filters.lancedb - assert "session_id = 'ses''s'" in filters.lancedb + milvus = render_milvus(filters) + lancedb = render_lance(filters) + assert 'owner_id == "al\'ice"' in milvus + assert 'session_id == "ses\'s"' in milvus + assert "owner_id = 'al''ice'" in lancedb + assert "session_id = 'ses''s'" in lancedb diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 9ee62af13..a84980d51 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -28,6 +28,7 @@ from everos.component.embedding import EmbeddingCapability from everos.component.rerank import RerankCapability from everos.core.errors import ProviderNotConfiguredError +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.search.dto import SearchMethod, SearchRequest from everos.memory.search.manager import SearchManager @@ -108,13 +109,13 @@ def __init__(self, sparse: list[Candidate], dense: list[Candidate]) -> None: async def sparse_recall( self, query: str, where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = str(where) + self.last_where = render_predicate(where) return list(self._sparse[:limit]) async def dense_recall( self, vector: Sequence[float], where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = str(where) + self.last_where = render_predicate(where) return list(self._dense[:limit]) async def fetch_by_parent_ids( diff --git a/tests/unit/test_memory/test_search/test_recall_agent_skill.py b/tests/unit/test_memory/test_search/test_recall_agent_skill.py index 9239fa34a..661e58a0b 100644 --- a/tests/unit/test_memory/test_search/test_recall_agent_skill.py +++ b/tests/unit/test_memory/test_search/test_recall_agent_skill.py @@ -26,6 +26,7 @@ agent_skill_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.agent_skill import AgentSkillRecaller from everos.memory.search.recall.base import RecallerDeps @@ -76,7 +77,7 @@ def _recaller() -> AgentSkillRecaller: return AgentSkillRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) -_OWNER_WHERE = "owner_id = 'agt' AND owner_type = 'agent'" +_OWNER_WHERE = all_of(eq("owner_id", "agt"), eq("owner_type", "agent")) async def test_fetch_by_case_ids_matches_any_lineage_case() -> None: diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index c3c71d433..6e81f755f 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -29,6 +29,7 @@ atomic_fact_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall import atomic_fact as atomic_fact_module from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -84,6 +85,9 @@ def _recaller() -> AtomicFactRecaller: return AtomicFactRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) +_ALICE_WHERE = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) + + async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: """Two episodes sharing one memcell both see the same fact pool. @@ -105,8 +109,9 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: "alice_ep_b": ["mc_shared"], "alice_ep_c": ["mc_other"], } - where = "owner_id = 'alice' AND owner_type = 'user'" - out = await _recaller().facts_for_episodes(ep_to_parents, where, per_episode=10) + out = await _recaller().facts_for_episodes( + ep_to_parents, _ALICE_WHERE, per_episode=10 + ) assert sorted(out.keys()) == ["alice_ep_a", "alice_ep_b", "alice_ep_c"] assert sorted(f.id for f in out["alice_ep_a"]) == ["alice_af_1", "alice_af_2"] @@ -121,9 +126,7 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: async def test_facts_for_episodes_returns_empty_for_no_episodes() -> None: - out: dict = await _recaller().facts_for_episodes( - {}, "owner_id = 'alice'", per_episode=10 - ) + out: dict = await _recaller().facts_for_episodes({}, _ALICE_WHERE, per_episode=10) assert out == {} @@ -135,7 +138,7 @@ async def test_facts_for_episodes_skips_unknown_memcells() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"], "alice_ep_b": ["mc_missing"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert "alice_ep_a" in out @@ -164,7 +167,7 @@ async def test_facts_for_episodes_filters_by_where_clause() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert [f.id for f in out["alice_ep_a"]] == ["alice_af_1"] @@ -184,7 +187,7 @@ async def test_facts_for_episodes_drops_empty_parent_ids() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": [""]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert out == {} @@ -232,7 +235,7 @@ async def test_facts_for_episodes_assigns_real_cosine_score_with_query_vector() out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_shared"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, query_vector=_unit_vector(0), ) @@ -260,7 +263,7 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, # no query_vector ) @@ -287,7 +290,7 @@ async def fake_dense_search(vector, where, *, limit): # type: ignore[no-untyped out = await _recaller().facts_for_episodes( ep_to_parents, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=20, query_vector=_unit_vector(0), ) @@ -316,7 +319,7 @@ async def test_facts_for_episodes_dual_parent_id_finds_both_eras() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["ep_entry_1", "mc_1"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) @@ -344,7 +347,7 @@ async def test_facts_for_episodes_multiple_parent_ids_dedup_across_episodes() -> "alice_ep_a": ["ep_entry_1", "mc_shared"], "alice_ep_b": ["ep_entry_2", "mc_shared"], }, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 2cde9f368..0ae51992c 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -8,6 +8,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.predicate import eq from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.episode import EpisodeRecaller @@ -42,6 +43,9 @@ def _mock_table(rows: list[dict[str, Any]]) -> MagicMock: return tbl +_ALICE_WHERE = eq("owner_id", "alice") + + @pytest.fixture() def recaller() -> EpisodeRecaller: tok = MagicMock(spec=Tokenizer) @@ -62,7 +66,7 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -79,7 +83,7 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result[0].metadata["episode_id"] == "ep_abc" assert result[0].metadata["parent_id"] == "mc_xyz" @@ -108,7 +112,7 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result == [] @@ -135,7 +139,7 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 1 assert result[0].id == "entry_xyz", "merged episode id must be entry_id" @@ -160,7 +164,7 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -184,7 +188,7 @@ async def test_fetch_by_entry_ids_returns_candidates( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids(["entry_xyz"], _ALICE_WHERE) assert len(result) == 1 assert result[0].id == "ep_merged" @@ -194,7 +198,7 @@ async def test_fetch_by_entry_ids_empty_input_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty entry_ids list short-circuits without querying.""" - result = await recaller.fetch_by_entry_ids([], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids([], _ALICE_WHERE) assert result == [] @@ -226,9 +230,7 @@ async def test_sparse_recall_as_child_injects_parent_id( new_callable=AsyncMock, return_value=rows, ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 assert result[0].metadata["parent_id"] == "entry_1" @@ -256,9 +258,7 @@ async def test_sparse_recall_as_child_falls_back_to_id_when_no_entry_id( new_callable=AsyncMock, return_value=[row], ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 cand = result[0] @@ -270,7 +270,7 @@ async def test_sparse_recall_as_child_empty_query_returns_empty( ) -> None: """Empty query token list short-circuits; no table call needed.""" recaller._deps.tokenizer.tokenize.return_value = [] - result = await recaller.sparse_recall_as_child("", "owner_id = 'alice'", limit=10) + result = await recaller.sparse_recall_as_child("", _ALICE_WHERE, limit=10) assert result == [] @@ -287,7 +287,7 @@ async def test_dense_recall_as_child_injects_parent_id( return_value=rows, ): result = await recaller.dense_recall_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -298,7 +298,7 @@ async def test_dense_recall_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_as_child([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_as_child([], _ALICE_WHERE, limit=10) assert result == [] @@ -321,31 +321,31 @@ async def test_dense_recall_subject_returns_subject_vector_source( rows = [ {**_make_row("ep_s1", "mc_s1", entry_id="entry_s1"), "_distance": 0.2}, ] - with ( - patch( - "everos.memory.search.recall.episode.active_backend", - return_value="lancedb", - ), - patch( - "everos.infra.persistence.lancedb.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), - ), - ): + with patch( + "everos.memory.search.recall.episode.episode_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, + ) as dense_search: result = await recaller.dense_recall_subject( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 assert result[0].source == "vector" assert result[0].score == pytest.approx(0.8) + dense_search.assert_awaited_once_with( + [0.1] * 1024, + _ALICE_WHERE, + limit=10, + vector_field="subject_vector", + ) async def test_dense_recall_subject_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_subject([], _ALICE_WHERE, limit=10) assert result == [] @@ -356,19 +356,13 @@ async def test_dense_recall_subject_as_child_injects_parent_id( rows = [ {**_make_row("ep_s2", "mc_s2", entry_id="entry_s2"), "_distance": 0.15}, ] - with ( - patch( - "everos.memory.search.recall.episode.active_backend", - return_value="lancedb", - ), - patch( - "everos.infra.persistence.lancedb.get_table", - new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), - ), + with patch( + "everos.memory.search.recall.episode.episode_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, ): result = await recaller.dense_recall_subject_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -380,7 +374,5 @@ async def test_dense_recall_subject_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject_as_child( - [], "owner_id = 'alice'", limit=10 - ) + result = await recaller.dense_recall_subject_as_child([], _ALICE_WHERE, limit=10) assert result == [] diff --git a/tests/unit/test_memory/test_search/test_recall_or_semantics.py b/tests/unit/test_memory/test_search/test_recall_or_semantics.py index 3e35cde83..0c2fe4a7f 100644 --- a/tests/unit/test_memory/test_search/test_recall_or_semantics.py +++ b/tests/unit/test_memory/test_search/test_recall_or_semantics.py @@ -30,6 +30,7 @@ episode_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.base import RecallerDeps, build_or_query from everos.memory.search.recall.episode import EpisodeRecaller @@ -148,7 +149,7 @@ async def test_or_semantics_poison_token_does_not_kill_query() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("alice support group", where, limit=10) assert cands, "alice + support + group should recall ep_1 via SHOULD" # ep_1 is the support-group episode; should rank above ep_2 (no support). @@ -177,7 +178,7 @@ async def test_or_semantics_single_informative_token() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("painting", where, limit=10) assert cands, "single informative token must recall the matching episode" assert cands[0].id == "alice_ep_2" @@ -185,5 +186,5 @@ async def test_or_semantics_single_informative_token() -> None: async def test_or_semantics_empty_query_returns_empty() -> None: """Tokenisation yields nothing → recall returns ``[]`` without hitting LanceDB.""" - cands = await _recaller().sparse_recall(" ", "owner_id = 'alice'", limit=10) + cands = await _recaller().sparse_recall(" ", eq("owner_id", "alice"), limit=10) assert cands == [] diff --git a/tests/unit/test_service/test_knowledge_search.py b/tests/unit/test_service/test_knowledge_search.py index 73500924f..037b39bd6 100644 --- a/tests/unit/test_service/test_knowledge_search.py +++ b/tests/unit/test_service/test_knowledge_search.py @@ -17,6 +17,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.errors import ProviderNotConfiguredError +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.sqlite.tables.knowledge import ( KnowledgeDocumentRow, ) @@ -134,11 +135,11 @@ def _patch_stack( class TestCompileKnowledgeWhere: def test_basic_clause(self) -> None: - result = compile_knowledge_where("myapp", "myproj") - assert result == "app_id = 'myapp' AND project_id = 'myproj'" + result = render_predicate(compile_knowledge_where("myapp", "myproj")) + assert result == "((app_id = 'myapp') AND (project_id = 'myproj'))" def test_defaults(self) -> None: - result = compile_knowledge_where("default", "default") + result = render_predicate(compile_knowledge_where("default", "default")) assert "app_id = 'default'" in result assert "project_id = 'default'" in result @@ -159,12 +160,12 @@ def test_rejects_empty_project_id(self) -> None: compile_knowledge_where("app", "") def test_accepts_valid_ids_with_special_chars(self) -> None: - result = compile_knowledge_where("my_app.v2", "project-1") + result = render_predicate(compile_knowledge_where("my_app.v2", "project-1")) assert "my_app.v2" in result assert "project-1" in result def test_accepts_valid_ids_with_at_plus(self) -> None: - result = compile_knowledge_where("app@org+v1", "proj_1") + result = render_predicate(compile_knowledge_where("app@org+v1", "proj_1")) assert "app@org+v1" in result assert "proj_1" in result diff --git a/uv.lock b/uv.lock index c9c12429c..f0757aa71 100644 --- a/uv.lock +++ b/uv.lock @@ -109,11 +109,11 @@ wheels = [ [[package]] name = "cachetools" -version = "7.1.6" +version = "7.1.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/55/af/861ebc2e318a5c3300e3eb63bc4d30f3d70a46d13b360093728ac0705eed/cachetools-7.1.6.tar.gz", hash = "sha256:c7a79e7f30ba9943c1cefd08cc36f006aaae086e017af9166f1d59d6170c47e1", size = 40572, upload-time = "2026-07-23T22:47:53.737Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/d2/47e8bc06fe2a06d3f5bdf20f1126ab66c4e99dc48d940e7ba873f7ac7131/cachetools-7.1.7.tar.gz", hash = "sha256:a3e2a00b14d8f8a6b70c1dae7b4685e7ad3bc965c5b42124a2d6ce895da6cf50", size = 40680, upload-time = "2026-08-01T21:20:40.434Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/f2/2086ba18a925a73586c4d4e61d25f4a6058e56fd00d77ce8f1d361ab4c9b/cachetools-7.1.6-py3-none-any.whl", hash = "sha256:2c12e255780330af28b91bb7fb96cce4c766f04e38396b9a24510190a5827096", size = 16954, upload-time = "2026-07-23T22:47:52.397Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d8/767faeda872075724b95dd675466a645f1b92aadcdcf2d1429dcfd76c176/cachetools-7.1.7-py3-none-any.whl", hash = "sha256:ef98ef375ad188819ef2f9b3645e3987f4b8c5b7550e436ad998c2de78296df0", size = 16830, upload-time = "2026-08-01T21:20:38.977Z" }, ] [[package]] @@ -613,7 +613,7 @@ dependencies = [ [package.optional-dependencies] milvus = [ - { name = "pymilvus", extra = ["milvus-lite"] }, + { name = "pymilvus" }, ] multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, @@ -631,6 +631,7 @@ dev = [ { name = "opentelemetry-sdk" }, { name = "pre-commit" }, { name = "pyinstrument" }, + { name = "pymilvus" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -661,7 +662,7 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, - { name = "pymilvus", extras = ["milvus-lite"], marker = "extra == 'milvus'", specifier = ">=3.0.0" }, + { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.0" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -682,6 +683,7 @@ dev = [ { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, { name = "pre-commit", specifier = ">=4.0.0" }, { name = "pyinstrument", specifier = ">=5.0.0" }, + { name = "pymilvus", specifier = ">=3.0.0" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.0.0" }, @@ -698,23 +700,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, ] -[[package]] -name = "faiss-cpu" -version = "1.14.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "packaging" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, - { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" }, - { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" }, - { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" }, - { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" }, - { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" }, -] - [[package]] name = "fastapi" version = "0.139.0" @@ -1365,21 +1350,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "milvus-lite" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "faiss-cpu" }, - { name = "grpcio" }, - { name = "numpy" }, - { name = "pyarrow" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/7f/58/54c72981958073d365febeae8fed3115c87761f45905f2d4725f506b94d1/milvus_lite-3.1.0.tar.gz", hash = "sha256:8d467ae81173f1ede93723a80f08d85b91988b451cd2b7bbf9b5a7cfb875e04b", size = 636180, upload-time = "2026-07-15T06:40:58.177Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/c2/fb5c2b73618973f943c8064e2d1f076f81731e2fafd5575a6ddaf3deda68/milvus_lite-3.1.0-py3-none-any.whl", hash = "sha256:0d20fffbee3c683eca69c4732a091273123368452159581ed050c933b41a59be", size = 252267, upload-time = "2026-07-15T06:40:52.538Z" }, -] - [[package]] name = "nodeenv" version = "1.10.0" @@ -1542,55 +1512,54 @@ wheels = [ [[package]] name = "orjson" -version = "3.11.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/0c/964746fcafbd16f8ff53219ad9f6b412b34f345c75f384ad434ceaadb538/orjson-3.11.9.tar.gz", hash = "sha256:4fef17e1f8722c11587a6ef18e35902450221da0028e65dbaaa543619e68e48f", size = 5599163, upload-time = "2026-05-06T15:11:08.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/6d/11867a3ffa3a3608d84a4de51ef4dd0896d6b5cc9132fbe1daf593e677bc/orjson-3.11.9-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9ef6fe90aadef185c7b128859f40beb24720b4ecea95379fc9000931179c3a49", size = 228515, upload-time = "2026-05-06T15:09:57.265Z" }, - { url = "https://files.pythonhosted.org/packages/24/75/05912954c8b288f34fcf5cd4b9b071cb4f6e77b9961e175e56ebb258089f/orjson-3.11.9-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:e5c9b8f28e726e97d97696c826bc7bea5d71cecd63576dba92924a32c1961291", size = 128409, upload-time = "2026-05-06T15:09:59.063Z" }, - { url = "https://files.pythonhosted.org/packages/ab/86/1c3a47df3bc8191ea9ac51603bbb872a95167a364320c269f2557911f406/orjson-3.11.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26a473dbb4162108b27901492546f83c76fdcea3d0eadff00ae7a07e18dcce09", size = 132106, upload-time = "2026-05-06T15:10:00.798Z" }, - { url = "https://files.pythonhosted.org/packages/d7/cf/b33b5f3e695ae7d63feef9d915c37cc3b8f465493dcd4f8e0b4c697a2366/orjson-3.11.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:011382e2a60fda9d46f1cdee31068cfc52ffe952b587d683ec0463002802a0f4", size = 127864, upload-time = "2026-05-06T15:10:02.15Z" }, - { url = "https://files.pythonhosted.org/packages/31/6a/6cf69385a58208024fcb8c014e2141b8ce838aba6492b589f8acfff97fab/orjson-3.11.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c2d3dc759490128c5c1711a53eeaa8ee1d437fd0038ffd2b6008abf46db3f882", size = 135213, upload-time = "2026-05-06T15:10:03.515Z" }, - { url = "https://files.pythonhosted.org/packages/e8/f8/0b1bd3e8f2efcdd376af5c8cfd79eaf13f018080c0089c80ebd724e3c7fb/orjson-3.11.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8ea516b3726d190e1b4297e6f4e7a8650347ae053868a18163b4dd3641d1fff", size = 145994, upload-time = "2026-05-06T15:10:05.083Z" }, - { url = "https://files.pythonhosted.org/packages/f3/59/dab79f61044c529d2c81aecdc589b1f833a1c8dec11ba3b1c2498a02ca7e/orjson-3.11.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380cdce7ba24989af81d0a7013d0aaec5d0e2a21734c0e2681b1bc4f141957fe", size = 132744, upload-time = "2026-05-06T15:10:06.853Z" }, - { url = "https://files.pythonhosted.org/packages/0e/a4/82b7a2fe5d8a67a59ed831b24d59a3d46ea7d207b66e1602d376541d94a6/orjson-3.11.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be4fa4f0af7fa18951f7ab3fc2148e223af211bf03f59e1c6034ec3f97f21d61", size = 134014, upload-time = "2026-05-06T15:10:08.213Z" }, - { url = "https://files.pythonhosted.org/packages/50/c7/375e83a76851b73b2e39f3bcf0e5a19e2b89bad13e5bca97d0b293d27f24/orjson-3.11.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a8f5f8bc7ce7d59f08d9f99fa510c06496164a24cb5f3d34537dbd9ca30132e2", size = 141509, upload-time = "2026-05-06T15:10:09.595Z" }, - { url = "https://files.pythonhosted.org/packages/7f/7c/49d5d82a3d3097f641f094f552131f1e2723b0b8cb0fa2874ab65ecfffa6/orjson-3.11.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4d7fde5501b944f83b3e665e1b31343ff6e154b15560a16b7130ea1e594a4206", size = 415127, upload-time = "2026-05-06T15:10:11.049Z" }, - { url = "https://files.pythonhosted.org/packages/3a/dc/7446c538590d55f455647e5f3c61fc33f7108714e7afcffa6a2a033f8350/orjson-3.11.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cde1a448023ba7d5bb4c01c5afb48894380b5e4956e0627266526587ef4e535f", size = 148025, upload-time = "2026-05-06T15:10:12.842Z" }, - { url = "https://files.pythonhosted.org/packages/df/e5/4d2d8af06f788329b4f78f8cc3679bb395392fcaa1e4d8d3c33e85308fa4/orjson-3.11.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e63adb0e1f1ed5d9e168f50a91ceb93ae6420731d222dc7da5c69409aa47aa", size = 136943, upload-time = "2026-05-06T15:10:14.405Z" }, - { url = "https://files.pythonhosted.org/packages/06/69/850264ccf6d80f6b174620d30a87f65c9b1490aba33fe6b62798e618cad3/orjson-3.11.9-cp312-cp312-win32.whl", hash = "sha256:2d057a602cdd19a0ad680417527c45b6961a095081c0f46fe0e03e304aac6470", size = 131606, upload-time = "2026-05-06T15:10:15.791Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d5/973a43fc9c55e20f2051e9830997649f669be0cb3ca52192087c0143f118/orjson-3.11.9-cp312-cp312-win_amd64.whl", hash = "sha256:59e403b1cc5a676da8eaf31f6254801b7341b3e29efa85f92b48d272637e77be", size = 127101, upload-time = "2026-05-06T15:10:17.129Z" }, - { url = "https://files.pythonhosted.org/packages/fe/ae/495470f0e4a18f73fa10b7f6b84b464ec4cc5291c4e0c7c2a6c400bef006/orjson-3.11.9-cp312-cp312-win_arm64.whl", hash = "sha256:9af678d6488357948f1f84c6cd1c1d397c014e1ae2f98ae082a44eb48f602624", size = 126736, upload-time = "2026-05-06T15:10:18.645Z" }, - { url = "https://files.pythonhosted.org/packages/32/33/93fcc25907235c344ae73122f8a4e01d2d393ef062b4af7d2e2487a32c37/orjson-3.11.9-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:4bab1b2d6141fe7b32ae71dac905666ece4f94936efbfb13d55bb7739a3a6021", size = 228458, upload-time = "2026-05-06T15:10:20.079Z" }, - { url = "https://files.pythonhosted.org/packages/8f/27/b1e6dadb3c080313c03fdd8067b85e6a0460c7d8d6a1c3984ef77b904e4d/orjson-3.11.9-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:844417969855fc7a41be124aafe83dc424592a7f77cd4501900c67307122b92c", size = 128368, upload-time = "2026-05-06T15:10:21.549Z" }, - { url = "https://files.pythonhosted.org/packages/21/0f/c9ede0bf052f6b4051e64a7d4fa91b725cccf8321a6a786e86eb03519f00/orjson-3.11.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffe02797b5e9f3a9d8292ddcd289b474ad13e81ad83cd1891a240811f1d2cb81", size = 132070, upload-time = "2026-05-06T15:10:23.371Z" }, - { url = "https://files.pythonhosted.org/packages/fd/26/d398e28048dc18205bbe812f2c88cb9b40313db2470778e25964796458fe/orjson-3.11.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e4eed3b200023042814d2fc8a5d2e880f13b52e1ed2485e83da4f3962f7dc1a", size = 127892, upload-time = "2026-05-06T15:10:24.714Z" }, - { url = "https://files.pythonhosted.org/packages/66/60/52b0054c4c700d5aa7fc5b7ca96917400d8f061307778578e67a10e25852/orjson-3.11.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aff7da9952a5ad1cef8e68017724d96c7b9a66e99e91d6252e1b133d67a7b10", size = 135217, upload-time = "2026-05-06T15:10:26.084Z" }, - { url = "https://files.pythonhosted.org/packages/d5/97/1e3dc2b2a28b7b2528f403d2fc1d79ec5f39af3bc143ab65d3ec26426385/orjson-3.11.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d4e98d6f3b8afed8bc8cd9718ec0cdf46661826beefb53fe8eafb37f2bf0362", size = 145980, upload-time = "2026-05-06T15:10:28.062Z" }, - { url = "https://files.pythonhosted.org/packages/fc/39/31fbfe7850f2de32dee7e7e5c09f26d403ab01e440ac96001c6b01ad3c99/orjson-3.11.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a81d52442a7c99b3662333235b3adf96a1715864658b35bb797212be7bddb97", size = 132738, upload-time = "2026-05-06T15:10:29.727Z" }, - { url = "https://files.pythonhosted.org/packages/a1/08/dca0082dd2a194acb93e5457e73455388e2e2ca464a2672449a9ddbb679d/orjson-3.11.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e39364e726a8fff737309aff059ff67d8a8c8d5b677be7bb49a8b3e84b7e218", size = 134033, upload-time = "2026-05-06T15:10:31.152Z" }, - { url = "https://files.pythonhosted.org/packages/11/d4/5bdb0626801230139987385554c5d4c42255218ac906525bf4347f22cd95/orjson-3.11.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4fd66214623f1b17501df9f0543bef0b833979ab5b6ded1e1d123222866aa8c9", size = 141492, upload-time = "2026-05-06T15:10:32.641Z" }, - { url = "https://files.pythonhosted.org/packages/fa/88/a21fb53b3ede6703aede6dce4710ed4111e5b201cfa6bbff5e544f9d47d7/orjson-3.11.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8ecc30f10465fa1e0ce13fd01d9e22c316e5053a719a8d915d4545a09a5ff677", size = 415087, upload-time = "2026-05-06T15:10:34.438Z" }, - { url = "https://files.pythonhosted.org/packages/3d/57/1b30daf70f0d8180e9a73cefbfbdd99e4bf19eb020466502b01fba7e0e50/orjson-3.11.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:97db4c94a7db398a5bd636273324f0b3fd58b350bbbac8bb380ceb825a9b40f4", size = 148031, upload-time = "2026-05-06T15:10:36.358Z" }, - { url = "https://files.pythonhosted.org/packages/04/83/45fbb6d962e260807f99441db9613cee868ceda4baceda59b3720a563f97/orjson-3.11.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9f78cf8fec5bd627f4082b8dfeac7871b43d7f3274904492a43dab39f18a19a0", size = 136915, upload-time = "2026-05-06T15:10:38.013Z" }, - { url = "https://files.pythonhosted.org/packages/5f/cc/2d10025f9056d376e4127ec05a5808b218d46f035fdc08178a5411b34250/orjson-3.11.9-cp313-cp313-win32.whl", hash = "sha256:d4087e5c0209a0a8efe4de3303c234b9c44d1174161dcd851e8eea07c7560b32", size = 131613, upload-time = "2026-05-06T15:10:39.569Z" }, - { url = "https://files.pythonhosted.org/packages/67/bd/2775ff28bfe883b9aa1ff348300542eb2ef1ee18d8ae0e3a49846817a865/orjson-3.11.9-cp313-cp313-win_amd64.whl", hash = "sha256:051b102c93b4f634e89f3866b07b9a9a98915ada541f4ec30f177067b2694979", size = 127086, upload-time = "2026-05-06T15:10:41.262Z" }, - { url = "https://files.pythonhosted.org/packages/91/2b/d26799e580939e32a7da9a39531bc9e58e15ca32ffaa6a8cb3e9bb0d22cd/orjson-3.11.9-cp313-cp313-win_arm64.whl", hash = "sha256:cce9127885941bd28f080cecf1f1d288336b7e0d812c345b08be88b572796254", size = 126696, upload-time = "2026-05-06T15:10:42.651Z" }, - { url = "https://files.pythonhosted.org/packages/8e/eb/5da01e356015aee6ecfa1187ced87aef51364e306f5e695dd52719bf0e78/orjson-3.11.9-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:b6ef1979adc4bc243523f1a2ba91418030a8e29b0a99cbe7e0e2d6807d4dce6e", size = 228465, upload-time = "2026-05-06T15:10:44.097Z" }, - { url = "https://files.pythonhosted.org/packages/64/62/3e0e0c14c957133bcd855395c62b55ed4e3b0af23ffea11b032cb1dcbdb1/orjson-3.11.9-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:f36b7f32c7c0db4a719f1fc5824db4a9c6f8bd1a354debb91faf26ebf3a4c71e", size = 128364, upload-time = "2026-05-06T15:10:45.839Z" }, - { url = "https://files.pythonhosted.org/packages/5a/5a/07d8aa117211a8ed7630bda80c8c0b14d04e0f8dcf99bcf49656e4a710eb/orjson-3.11.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08f4d8ebb44925c794e535b2bebc507cebf32209df81de22ae285fb0d8d66de0", size = 132063, upload-time = "2026-05-06T15:10:47.267Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ec/4acaf21483e18aa945be74a474c74b434f284b549f275a0a39b9f98956e9/orjson-3.11.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6cc7923789694fd58f001cbcac7e47abc13af4d560ebbfcf3b41a8b1a0748124", size = 122356, upload-time = "2026-05-06T15:10:48.765Z" }, - { url = "https://files.pythonhosted.org/packages/13/d8/5f0555e7638801323b7a75850f92e7dfa891bc84fe27a1ba4449170d1200/orjson-3.11.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea5c46eb2d3af39e806b986f4b09d5c2706a1f5afde3cbf7544ce6616127173c", size = 129592, upload-time = "2026-05-06T15:10:50.13Z" }, - { url = "https://files.pythonhosted.org/packages/b6/30/ed9860412a3603ceb3c5955bfd72d28b9d0e7ba6ed81add14f83d7114236/orjson-3.11.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f5d89a2ed90731df3be64bab0aa44f78bff39fdc9d71c291f4a8023aa46425b7", size = 140491, upload-time = "2026-05-06T15:10:51.582Z" }, - { url = "https://files.pythonhosted.org/packages/d0/17/adc514dea7ac7c505527febf884934b815d34f0c7b8693c1a8b39c5c4a57/orjson-3.11.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25e4aed0312d292c09f61af25bba34e0b2c88546041472b09088c39a4d828af1", size = 127309, upload-time = "2026-05-06T15:10:53.329Z" }, - { url = "https://files.pythonhosted.org/packages/76/3e/c0b690253f0b82d86e99949af13533363acfb5432ecb5d53dd5b3bce9c34/orjson-3.11.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aaea64f3f467d22e70eeed68bdccb3bc4f83f650446c4a03c59f2cba28a108db", size = 134030, upload-time = "2026-05-06T15:10:54.988Z" }, - { url = "https://files.pythonhosted.org/packages/c1/7a/bc82a0bb25e9faaf92dc4d9ef002732efc09737706af83e346788641d4a7/orjson-3.11.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a028425d1b440c5d92a6be1e1a020739dfe67ea87d96c6dbe828c1b30041728b", size = 141482, upload-time = "2026-05-06T15:10:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/01/55/e69188b939f77d5d32a9833745ace31ea5ccae3ab613a1ec185d3cd2c4fb/orjson-3.11.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5b192c6cf397e4455b11523c5cf2b18ed084c1bbd61b6c0926344d2129481972", size = 415178, upload-time = "2026-05-06T15:10:58.446Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1a/b8a5a7ac527e80b9cb11d51e3f6689b709279183264b9ec5c7bc680bb8b5/orjson-3.11.9-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea407d4ccf5891d667d045fecae97a7a1e5e87b3b97f97ae1803c2e741130be0", size = 148089, upload-time = "2026-05-06T15:11:00.441Z" }, - { url = "https://files.pythonhosted.org/packages/97/4e/00503f64204bf859b37213a63927028f30fb6268cd8677fb0a5ad48155e1/orjson-3.11.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f63aaf97afd9f6dec5b1a68e1b8da12bfccb4cb9a9a65c3e0b6c847849e7586", size = 136921, upload-time = "2026-05-06T15:11:02.176Z" }, - { url = "https://files.pythonhosted.org/packages/0d/ba/a23b82a0a8d0ed7bed4e5f5035aae751cad4ff6a1e8d2ecd14d8860f5929/orjson-3.11.9-cp314-cp314-win32.whl", hash = "sha256:e30ab17845bb9fa54ccf67fa4f9f5282652d54faa6d17452f47d0f369d038673", size = 131638, upload-time = "2026-05-06T15:11:03.696Z" }, - { url = "https://files.pythonhosted.org/packages/f3/c3/0c6798456bade745c75c452342dabacce5798196483e77e643be1f53877d/orjson-3.11.9-cp314-cp314-win_amd64.whl", hash = "sha256:32ef5f4283a3be81913947d19608eacb7c6608026851123790cd9cc8982af34b", size = 127078, upload-time = "2026-05-06T15:11:05.123Z" }, - { url = "https://files.pythonhosted.org/packages/16/21/5a3f1e8913103b703a436a5664238e5b965ec392b555fe68943ea3691e6b/orjson-3.11.9-cp314-cp314-win_arm64.whl", hash = "sha256:eebdbdeef0094e4f5aefa20dcd4eb2368ab5e7a3b4edea27f1e7b2892e009cf9", size = 126687, upload-time = "2026-05-06T15:11:06.602Z" }, +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/4a/295da39c651c2faac8bd351a2a346f0fdedd9d50b847ee9dfc27d2207ef6/orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0", size = 223427, upload-time = "2026-08-14T16:12:28.525Z" }, + { url = "https://files.pythonhosted.org/packages/29/98/758cf90fbeaaafb7f8141bfac75a432099959f3a2f5db93a412e876415d8/orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54", size = 123725, upload-time = "2026-08-14T16:12:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/32/b5/5b934d251f8651f7e41df180ad0c57a6e1cabe15c7bd331638413a50ebc9/orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83", size = 113375, upload-time = "2026-08-14T16:12:31.209Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d2/37efb5b12a176ce3ced29f4144f20da57d02757f78ce549637dc1b4e1fc8/orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7", size = 129983, upload-time = "2026-08-14T16:12:32.721Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/0644b87c73f13e0092df8f35a1fe280d991e5e90072087411e0dd7e44e0c/orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e", size = 130629, upload-time = "2026-08-14T16:12:34.084Z" }, + { url = "https://files.pythonhosted.org/packages/8c/57/80b986ebfecd9c6a177ddf1c2319717f0cd8feffb2b78946595a18a2fc88/orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b", size = 131245, upload-time = "2026-08-14T16:12:35.713Z" }, + { url = "https://files.pythonhosted.org/packages/80/3d/75c5ac5a69161f44492a68fbdde66f4cc4ce48cd5e1fb05918e46f0c8848/orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f", size = 135397, upload-time = "2026-08-14T16:12:37.128Z" }, + { url = "https://files.pythonhosted.org/packages/71/93/4d71f2df314a97ff0d27a4559bf5888fc8406e3c6dec90e92291e3511215/orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873", size = 127693, upload-time = "2026-08-14T16:12:38.627Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/0dbc6be5adfd1730491072fb60beb6bcdf5d7b2596ee41b7fc2e298bfc09/orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5", size = 128000, upload-time = "2026-08-14T16:12:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c9/97b1ce0112ebf5e949c775ed5b1755e562233179f3584579673cc24d6378/orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a", size = 122106, upload-time = "2026-08-14T16:12:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6a/facd8b312e4a0d3a7fa978c7e15821f74a336adf1d65529faec33b48e18b/orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d", size = 126869, upload-time = "2026-08-14T16:12:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/54/cb/d7b78218a987eb8a8ce4eeae0286b1bb679333eb631ea0eeaf6371680bfc/orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900", size = 223397, upload-time = "2026-08-14T16:12:44.003Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/bc87c45e7ec639d35ebefd62618e01939531ac8e171426606a01bda05914/orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03", size = 123662, upload-time = "2026-08-14T16:12:45.433Z" }, + { url = "https://files.pythonhosted.org/packages/94/ee/c9a4ff3f2dbedbbe9e635d0fa72c8866adede09b6335ef9644f53752f0d8/orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8", size = 113374, upload-time = "2026-08-14T16:12:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/75/09/3f330a026a796c8b4c97a6f429652a5e912e7065039bf96ed25e42aa7b25/orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94", size = 130029, upload-time = "2026-08-14T16:12:48.06Z" }, + { url = "https://files.pythonhosted.org/packages/7d/40/094cc53126a3d22f76cdf83b6ea67338bed01d774037621a785aa8e6e5ea/orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806", size = 130528, upload-time = "2026-08-14T16:12:49.362Z" }, + { url = "https://files.pythonhosted.org/packages/bc/74/89bb236deb9565f99434b13052bb40ddfcce4adf3afbfa3132ee7e421468/orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df", size = 131075, upload-time = "2026-08-14T16:12:50.692Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ac/1176360d762c01b5bd34acd56fc098e936c491363d8b6b397ad4aa475547/orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978", size = 135321, upload-time = "2026-08-14T16:12:52.114Z" }, + { url = "https://files.pythonhosted.org/packages/7a/02/bbd881c8b9276d50b998de38b4e97de8ace1aac940b0ee545aedbf65ed00/orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222", size = 127472, upload-time = "2026-08-14T16:12:53.517Z" }, + { url = "https://files.pythonhosted.org/packages/8e/02/a0934d7503e6dcbedd6afac3e7f3f8597fd09389949ad94d0f7540e9dbca/orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1", size = 128000, upload-time = "2026-08-14T16:12:55.14Z" }, + { url = "https://files.pythonhosted.org/packages/52/87/69f98f8d40faff103a965a5fbb83f08241b01beaf92badb5413fbc9358cc/orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2", size = 121841, upload-time = "2026-08-14T16:12:56.507Z" }, + { url = "https://files.pythonhosted.org/packages/e6/07/b83046a4e3cadcc0987d0f160696107c4af706a619b56e4ad01940cadadf/orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e", size = 126765, upload-time = "2026-08-14T16:12:57.806Z" }, + { url = "https://files.pythonhosted.org/packages/12/9d/3931253e6f3148abf2cbe14830367042a4806b362ea520df2303db188fb9/orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d", size = 223391, upload-time = "2026-08-14T16:12:59.184Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/b4a4f1e305367245877b967a0bad70fcf001d77c54ac4339a120b66fdae4/orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647", size = 123659, upload-time = "2026-08-14T16:13:00.548Z" }, + { url = "https://files.pythonhosted.org/packages/96/f3/6782c6fa85e2702bc66be183c3b421486167dcf266ee4dc1403fe3824870/orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c", size = 113337, upload-time = "2026-08-14T16:13:02.009Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/b32ab64bacda9d0fa4942ef483bd03cabf0eaf2be819ca9fb7ff610c559d/orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc", size = 130112, upload-time = "2026-08-14T16:13:03.404Z" }, + { url = "https://files.pythonhosted.org/packages/ee/49/6e6142999ca01509219be5e5a9c338a3e5ea011f63e91ff473fbbf3734ed/orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1", size = 130520, upload-time = "2026-08-14T16:13:04.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/d0/3745af0a4cc9867784f29722929cec4d10bd1c877cd754b01ba6d96eb21a/orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a", size = 131053, upload-time = "2026-08-14T16:13:06.14Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f4/6fe5a22fa478fffb190e65c338c84df5c311ef597b363150a17cc57063c0/orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e", size = 135321, upload-time = "2026-08-14T16:13:07.544Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/b1b0ec30289646a81a76e2dbaae2686b96fcccb7cb0323dc1dd78cbc7875/orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f", size = 127485, upload-time = "2026-08-14T16:13:08.88Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2b/277404bdcc21c93b112b963655b76443ebfe828f8a3ff1de7d90f8850eb3/orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92", size = 128048, upload-time = "2026-08-14T16:13:10.305Z" }, + { url = "https://files.pythonhosted.org/packages/41/2b/395b36fa2b4ce7af70b651d715e88f80d884b2c2b14a6b53e84d554fb5f0/orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed", size = 121858, upload-time = "2026-08-14T16:13:11.634Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a3/833e895ff452859eebe75093d26691fe9108f1a7a6a08435d7a5780ea652/orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7", size = 126749, upload-time = "2026-08-14T16:13:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/58/64/99c8947ece10c17176af9aae85c4948f1d109da77440ec14d87239efaf73/orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e", size = 223398, upload-time = "2026-08-14T16:13:14.694Z" }, + { url = "https://files.pythonhosted.org/packages/3e/30/cf983fe09f2731420fda097a9f7ef4343f47fa216c228961ad8f6da44f3d/orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517", size = 123655, upload-time = "2026-08-14T16:13:16.221Z" }, + { url = "https://files.pythonhosted.org/packages/11/50/9cb8ae73fa4749dbbc20f617004213b5ff01c20aaeec34c3f31124f2c1d8/orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38", size = 130515, upload-time = "2026-08-14T16:13:17.601Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0a/adb6ce1a5b5fbf9cb1790f9961bb668a0dd5429aadaf6cee044724681795/orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d", size = 113327, upload-time = "2026-08-14T16:13:18.927Z" }, + { url = "https://files.pythonhosted.org/packages/51/5c/d17f61581d8dbdde7048f87a330fa24915edec38db4d72b381fec14fbb56/orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13", size = 130105, upload-time = "2026-08-14T16:13:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b7/938befcf33bee4704a92ecec6a2731224c539d939bf9429fd39396d28931/orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328", size = 131049, upload-time = "2026-08-14T16:13:21.719Z" }, + { url = "https://files.pythonhosted.org/packages/b0/15/cfa2021d64d5aa8bb5c9f604ef375e00ec8b657651b5dd650b1b7ad13df1/orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c", size = 135320, upload-time = "2026-08-14T16:13:23.415Z" }, + { url = "https://files.pythonhosted.org/packages/1a/50/3e75dfe357c1e8f9e287c7a5740260ef15bd23a5299eae8d0835dcad5375/orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a", size = 127488, upload-time = "2026-08-14T16:13:24.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/a6/79aed402eb3ab284dc5b4791a7ad62c5875127de01b8e3f04bd92d551298/orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55", size = 128048, upload-time = "2026-08-14T16:13:26.217Z" }, + { url = "https://files.pythonhosted.org/packages/64/f7/2723e264aab7248c1ed6ecaad8e5d0cb866c0cffde75442102ffa7491aba/orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578", size = 121860, upload-time = "2026-08-14T16:13:27.577Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/630c9113ec8996778f1f0304b364b091b9a9db5fef5fdc17cca622f5ea24/orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc", size = 126754, upload-time = "2026-08-14T16:13:28.962Z" }, ] [[package]] @@ -2073,7 +2042,7 @@ wheels = [ [[package]] name = "pymilvus" -version = "3.0.0" +version = "3.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -2083,15 +2052,10 @@ dependencies = [ { name = "protobuf" }, { name = "python-dotenv" }, { name = "requests" }, - { name = "setuptools" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/c0/78/6bd0dba340706bc63346af96f0ebe36ff17f75404f5de935fda94d476c98/pymilvus-3.0.1.tar.gz", hash = "sha256:c02389059088b18d6e598cd175541e445c772fab4926c5e527c4913be34887f1", size = 347593, upload-time = "2026-07-29T14:55:43.061Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/c1/01647e61f3a82fd881382746b6dde3401d65b88cd4f75bd059901fb2392b/pymilvus-3.0.0-1-py3-none-any.whl", hash = "sha256:57c8e7c87fbbf579f122b4df893949bc78e50bca2988527864891bd544817b05", size = 344817, upload-time = "2026-05-07T14:57:45.235Z" }, -] - -[package.optional-dependencies] -milvus-lite = [ - { name = "milvus-lite", marker = "sys_platform != 'win32'" }, + { url = "https://files.pythonhosted.org/packages/60/9d/7011887b29f452905745e8bd321f404068d5bfe78fe84e42c0b7cd81a065/pymilvus-3.0.1-py3-none-any.whl", hash = "sha256:c5a8d5c1fa1de7b416e3529d383d8cc2e7da2170433ffa4a2d9087e14f70171a", size = 386820, upload-time = "2026-07-29T14:55:44.279Z" }, ] [[package]] @@ -2399,15 +2363,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/98/6beb4b351e472e5f4c4613f7c35a5290b8be2497e183825310c4c3a3984b/ruff-0.15.12-py3-none-win_arm64.whl", hash = "sha256:a538f7a82d061cee7be55542aca1d86d1393d55d81d4fcc314370f4340930d4f", size = 11120821, upload-time = "2026-04-24T18:16:57.979Z" }, ] -[[package]] -name = "setuptools" -version = "83.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" From 79b629fd99e940418517ae7f33f9e5fe5d7ecc19 Mon Sep 17 00:00:00 2001 From: Cheney Zhang Date: Wed, 19 Aug 2026 04:02:51 +0000 Subject: [PATCH 5/5] docs(milvus): add remote connection examples Signed-off-by: Cheney Zhang --- docs/configuration.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 0608170e9..f30af50ca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -120,6 +120,32 @@ paths are not supported. | `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | | `collection_prefix` | string | `"everos"` | Prefix for EverOS collections. Must start with a letter or underscore and contain only letters, digits, and underscores. Make it unique when deployments share a database. | +For a self-hosted Milvus Server, a minimal configuration is: + +```toml +[index] +backend = "milvus" + +[milvus] +uri = "http://localhost:19530" +collection_prefix = "everos" +``` + +Zilliz Cloud uses the same backend. Credentials can be supplied through +environment variables: + +```bash +export EVEROS_INDEX__BACKEND=milvus +export EVEROS_MILVUS__URI="https://your-cluster-endpoint" +export EVEROS_MILVUS__TOKEN="" +export EVEROS_MILVUS__DB_NAME="default" +export EVEROS_MILVUS__COLLECTION_PREFIX="everos_prod" +``` + +Use a deployment secret rather than committing the token to a configuration +file. If the memory root already contains Markdown records, stop the EverOS +server and run `everos cascade rebuild --yes` to populate the derived index. + EverOS creates seven collections and supports every dense field in the shared index schema, including `episode.subject_vector`. The vector dimension is owned by that schema (currently 1024), rather than duplicated in Milvus config. @@ -261,3 +287,6 @@ Examples: | `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | | `[index] backend = "milvus"` | `EVEROS_INDEX__BACKEND=milvus` | | `[milvus] uri = "http://localhost:19530"` | `EVEROS_MILVUS__URI=http://localhost:19530` | +| `[milvus] token = "..."` | `EVEROS_MILVUS__TOKEN=...` | +| `[milvus] db_name = "default"` | `EVEROS_MILVUS__DB_NAME=default` | +| `[milvus] collection_prefix = "everos_prod"` | `EVEROS_MILVUS__COLLECTION_PREFIX=everos_prod` |