diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b90d55c9..7d1fdd3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,6 +126,49 @@ jobs: - name: Integration tests run: make integration + milvus-integration: + name: Milvus Server integration + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install uv + uses: astral-sh/setup-uv@v8.2.0 + with: + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Set up Python + run: uv python install 3.12 + + - name: Install dependencies (frozen) + run: make install-deps + + - name: Start Milvus Server + run: | + curl -fsSL \ + https://github.com/milvus-io/milvus/releases/download/v2.6.22/milvus-standalone-docker-compose.yml \ + -o /tmp/milvus-compose.yml + docker compose -f /tmp/milvus-compose.yml up -d + for attempt in $(seq 1 90); do + if curl -fsS http://127.0.0.1:9091/healthz; then + exit 0 + fi + sleep 2 + done + docker compose -f /tmp/milvus-compose.yml logs + exit 1 + + - name: Milvus derived-index contract + env: + EVEROS_TEST_MILVUS_URI: http://127.0.0.1:19530 + EVEROS_TEST_MILVUS_FULL_STARTUP: "1" + run: uv run pytest tests/integration/test_milvus_remote.py -v + + - name: Stop Milvus Server + if: always() + run: docker compose -f /tmp/milvus-compose.yml down -v + package: name: package build runs-on: ubuntu-latest diff --git a/config.example.toml b/config.example.toml index f8043127..00045b2a 100644 --- a/config.example.toml +++ b/config.example.toml @@ -56,3 +56,19 @@ max_concurrent = 5 # # [lancedb] # read_consistency_seconds = 5.0 + +# ── Optional Milvus derived index ───────────────────── +# Markdown remains the source of truth. Set the index backend to Milvus +# when you want the rebuildable vector/BM25 index to live in an external +# Milvus Server or Zilliz Cloud deployment. Embedded Milvus Lite and local +# database paths are not supported. +# +# [index] +# backend = "milvus" +# +# [milvus] +# uri = "http://localhost:19530" # required remote endpoint +# token = "" # required for Zilliz Cloud/auth-enabled Milvus +# db_name = "" # use a dedicated database when available +# consistency_level = "Session" +# collection_prefix = "everos" # make unique on a shared database diff --git a/docs/api.md b/docs/api.md index 422b47ff..d19361f4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -93,8 +93,9 @@ bare FastAPI `detail`); see [Errors](#errors). `/add` and `/flush` write the markdown file (the source of truth) **synchronously** — when the call returns with `status: "extracted"`, -the new entry exists on disk. The LanceDB vector / BM25 / scalar index -is rebuilt by the in-process **cascade coroutine asynchronously**. +the new entry exists on disk. The configured vector / BM25 / scalar +index backend is rebuilt by the in-process **cascade coroutine +asynchronously**. That means `/search` and `/get` may not see a record immediately after the `/flush` that produced it. Typical sync latency is sub-second, but @@ -371,7 +372,7 @@ A recursive boolean tree of predicates. Used by `/search.filters` and `/get.filters`. The Pydantic envelope only checks the recursive combinator shape; field-level validity (which scalar fields are filterable, which operators apply, value coercion) runs when the -node is compiled to a LanceDB `where` clause server-side. Compile +node is compiled to a backend-specific filter clause server-side. Compile errors surface as `422` with the offending field / operator in `error.message`. @@ -464,12 +465,12 @@ Examples: |---|---| | `"keyword"` | BM25 only — pure lexical match, no embedding cost | | `"vector"` | Dense vector ANN only — semantic recall, no lexical | -| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter in a single LanceDB query | +| `"hybrid"` *(default)* | Reciprocal-rank fuse of BM25 + vector + optional scalar filter against the configured derived index backend | | `"agentic"` | Iterative cluster-path retrieval driven by a cross-encoder rerank loop; higher quality at higher latency / cost | `"hybrid"` is the default because it balances recall and precision -with one LanceDB roundtrip. `"agentic"` calls the LLM in a loop and -should be reserved for offline or background workflows. +without requiring the agentic loop. `"agentic"` calls the LLM in a loop +and should be reserved for offline or background workflows. ### GetMemoryType @@ -613,7 +614,7 @@ scope. this `(session_id, app_id, project_id)`, or it was already flushed). `/flush` is synchronous with respect to markdown persistence: by the -time the response returns, the new entry is on disk. LanceDB index +time the response returns, the new entry is on disk. Derived index sync is still asynchronous — see [Eventual consistency](#eventual-consistency). diff --git a/docs/architecture.md b/docs/architecture.md index afb5efe3..4aae7826 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ │ + reflection + strategies + get + events │ ├──────────────────────────────────────────────────────┤ │ infra/persistence (Storage adapters; infra/ may host other adapter types) │ -│ markdown + sqlite + lancedb │ +│ markdown + sqlite + derived index │ └──────────────────────────────────────────────────────┘ Cross-cutting (used by all layers, depends on none): @@ -68,8 +68,8 @@ layers = [ └────────────────────────────────────────────────────────────────┘ ┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ - │ Markdown │ │ SQLite │ │ LanceDB │ - │ (truth) │ │ (state) │ │ (index) │ + │ Markdown │ │ SQLite │ │ Derived index │ + │ (truth) │ │ (state) │ │ LanceDB/Milvus │ ├──────────────┤ ├──────────────┤ ├─────────────────┤ │ entries + │ │ change queue │ │ vector ANN │ │ frontmatter │ │ + state/LSN │ │ BM25 (Tantivy) │ @@ -78,7 +78,7 @@ layers = [ └──────────────┘ └──────────────┘ └─────────────────┘ │ │ │ ▼ ▼ ▼ - memory-root/ .index/sqlite/ .index/lancedb/ + memory-root/ .index/sqlite/ .index// (truth source) (system data) (rebuildable) ``` @@ -101,10 +101,13 @@ External message │ │ ▼ ▼ 4a. SQLite 4b. memory.cascade (async daemon) - audit watches md → diff entries → LanceDB sync + audit watches md → diff entries → index sync ``` -**Key guarantee**: md write is strongly consistent (fsync). LanceDB is eventually consistent. LanceDB unavailability does not block response — changes buffer in the SQLite `md_change_state` queue, replayed on recovery. +**Key guarantee**: md write is strongly consistent (fsync). The derived +index is eventually consistent. Index backend unavailability does not block +response — changes buffer in the SQLite `md_change_state` queue, replayed on +recovery. ## Read path @@ -115,8 +118,8 @@ User query 1. service.search │ ▼ -2. memory.search (hybrid) single LanceDB query = - BM25 + vector ANN + scalar filter +2. memory.search (hybrid) BM25 + vector ANN + scalar filter + through the configured index backend │ ▼ 3. (optional) read md original markdown for context @@ -139,12 +142,13 @@ extract/ ### `memory/cascade/` -Daemon that watches markdown changes and syncs to LanceDB: +Daemon that watches markdown changes and syncs to the configured derived +index backend: - inotify / FSEvents file watcher (cross-platform via `watchdog`) - 500ms debounce - Entry-level diff (added / changed / removed) -- LanceDB single-transaction update (text + vector columns atomic) +- Per-entry index upsert / delete (text + vector columns update together) - LSN-based crash recovery via the SQLite `md_change_state` queue - Handlers for all eight business kinds: episode, atomic_fact, foresight, user_profile, agent_case, agent_skill, knowledge_document, knowledge_topic @@ -225,7 +229,7 @@ holding **only memory extraction algorithms**: everalgo is: - **Stateless** — pure functions, no class hierarchy -- **No I/O** — does not touch md files / LanceDB / SQLite +- **No I/O** — does not touch md files, derived indexes, or SQLite - **No prompts inline** — extractors that accept a prompt-override parameter use the project-supplied value; others use their algo-bundled defaults This boundary lets everalgo be reused across product forms (this open-source build, EverOS Cloud, OpenClaw plugins, etc.). diff --git a/docs/cascade_runbook.md b/docs/cascade_runbook.md index 3e0dcd90..338d2a7c 100644 --- a/docs/cascade_runbook.md +++ b/docs/cascade_runbook.md @@ -1,9 +1,13 @@ # Cascade Runbook -The cascade daemon keeps LanceDB in sync with the markdown files under -the memory root. Service / entry points only ever write markdown; the -daemon is the **sole** writer of the LanceDB index. This runbook covers -the recurring operational questions. +The cascade daemon keeps the configured derived index in sync with the +markdown files under the memory root. Service / entry points only ever write +markdown; the daemon is the **sole** writer of the derived index. This runbook +covers the recurring operational questions. + +Sections that mention LanceDB-specific schemas, index cache, file descriptors, +or `lance error` messages apply to the default LanceDB backend. Milvus uses the +same cascade queue with backend-specific collection management. ## What runs where @@ -13,7 +17,7 @@ providers in order: 1. **Metrics** — Prometheus collector. 2. **LLM** — LLM client initialisation. 3. **SQLite** — system DB + schema (`SQLModel.metadata.create_all`). -4. **LanceDB** — async connection + schema verification + FTS indexes. +4. **Derived index** — async connection + schema verification + search indexes. 5. **Cascade** — watcher + scanner + worker, all in-process tasks. 6. **OME** — offline memory engine. @@ -23,7 +27,7 @@ The cascade subsystem itself is three independent loops: |---|---|---| | Watcher | `watchdog` filesystem events (sync thread) | `md_change_state.upsert` per registered kind | | Scanner | Periodic walk (`scan_interval_seconds`, default 30 s) | Same — catches changes the watcher missed | -| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → LanceDB upsert / delete | +| Worker | `claim_pending_batch` polling (default 1 s when idle) | Handler dispatch → index upsert / delete | Every loop talks to the same `md_change_state` sqlite table. The worker's claim mode (`pending → processing → done/failed`) keeps @@ -123,7 +127,7 @@ parallel with a live `everos server`. ## Rebuild the index: `everos cascade rebuild` -The safe recovery from a drifted or corrupt LanceDB index. It rebuilds +The safe recovery from a drifted or corrupt derived index. It rebuilds the whole index from markdown (the source of truth) in one shot: ```bash @@ -132,15 +136,16 @@ everos cascade rebuild --yes # non-interactive ``` > **Stop the `everos server` first.** Unlike `cascade sync`, rebuild -> **drops and recreates** the LanceDB tables. A running daemon holds +> **drops and recreates** the active backend's tables or collections. A running +> daemon holds > cached table handles that would keep pointing at (and writing to) the > dropped dataset, corrupting the rebuild. This is the one cascade > command that is **not** safe to run alongside a live server. What it does, in order: -1. **Drops** every business LanceDB table (`drop_business_tables`) and - evicts them from the connection cache. +1. **Drops** every business table or collection (`drop_business_tables`) and + evicts it from the process cache. 2. **Recreates** them empty from the current schema + FTS indexes (`ensure_business_indexes`). 3. **Clears** the cascade queue (`md_change_state.reset_all`) so every @@ -160,6 +165,10 @@ Why not a bare `rm`: | `rm -rf .index` | ✅ | ❌ deletes un-extracted messages | | `everos cascade rebuild` | ✅ | ✅ | +For a remote Milvus backend, rebuild acts only on collections whose names use +the configured `collection_prefix`. Use a unique prefix or dedicated database +before running it on a shared Milvus Server or Zilliz Cloud deployment. + ## Recovery paths ### LanceDB schema drift on startup diff --git a/docs/configuration.md b/docs/configuration.md index 98fdf347..f30af50c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -99,6 +99,63 @@ everos init --root /data/everos | `read_consistency_seconds` | float \| null | `null` | Read consistency interval. `null` = no check, `0` = strict, `>0` = eventual. | | `index_cache_size_bytes` | int | `16777216` | Upper bound on LanceDB index cache (16 MB default). | +### `[index]` + +| Field | Type | Default | Description | +|---|---|---|---| +| `backend` | `"lancedb"` \| `"milvus"` | `"lancedb"` | Rebuildable vector/BM25 index backend used by cascade, search, and get. Markdown remains the source of truth; SQLite remains the system state store. | + +### `[milvus]` + +Milvus is optional. Install with `everos[milvus]`, then set +`[index].backend = "milvus"`. The backend connects only to an external +Milvus Server or Zilliz Cloud endpoint; embedded Milvus Lite and local `.db` +paths are not supported. + +| Field | Type | Default | Description | +|---|---|---|---| +| `uri` | string | `""` | Required when the Milvus backend is selected. Must be a remote Milvus Server or Zilliz Cloud endpoint. | +| `token` | string | `""` | Token for Zilliz Cloud or auth-enabled Milvus Server. | +| `db_name` | string | `""` | Optional Milvus database name. Prefer a dedicated database on shared clusters. | +| `consistency_level` | `"Strong"` \| `"Session"` \| `"Bounded"` \| `"Eventually"` | `"Session"` | Milvus consistency level for created collections. | +| `collection_prefix` | string | `"everos"` | Prefix for EverOS collections. Must start with a letter or underscore and contain only letters, digits, and underscores. Make it unique when deployments share a database. | + +For a self-hosted Milvus Server, a minimal configuration is: + +```toml +[index] +backend = "milvus" + +[milvus] +uri = "http://localhost:19530" +collection_prefix = "everos" +``` + +Zilliz Cloud uses the same backend. Credentials can be supplied through +environment variables: + +```bash +export EVEROS_INDEX__BACKEND=milvus +export EVEROS_MILVUS__URI="https://your-cluster-endpoint" +export EVEROS_MILVUS__TOKEN="" +export EVEROS_MILVUS__DB_NAME="default" +export EVEROS_MILVUS__COLLECTION_PREFIX="everos_prod" +``` + +Use a deployment secret rather than committing the token to a configuration +file. If the memory root already contains Markdown records, stop the EverOS +server and run `everos cascade rebuild --yes` to populate the derived index. + +EverOS creates seven collections and supports every dense field in the shared +index schema, including `episode.subject_vector`. The vector dimension is owned +by that schema (currently 1024), rather than duplicated in Milvus config. + +Milvus imposes limits that LanceDB does not: strings are capped at 65,535 +UTF-8 bytes, string arrays at 256 items, and each array item at 512 UTF-8 +bytes. EverOS validates these before a write and reports the exact table and +field instead of relying on a server-side insert error. Ensure the selected +Zilliz Cloud plan permits at least seven collections. + ### `[llm]` | Field | Type | Default | Required | Description | @@ -228,3 +285,8 @@ Examples: | `[llm] api_key = "sk-..."` | `EVEROS_LLM__API_KEY=sk-...` | | `[sqlite] busy_timeout_ms = 10000` | `EVEROS_SQLITE__BUSY_TIMEOUT_MS=10000` | | `[memory] timezone = "Asia/Tokyo"` | `EVEROS_MEMORY__TIMEZONE=Asia/Tokyo` | +| `[index] backend = "milvus"` | `EVEROS_INDEX__BACKEND=milvus` | +| `[milvus] uri = "http://localhost:19530"` | `EVEROS_MILVUS__URI=http://localhost:19530` | +| `[milvus] token = "..."` | `EVEROS_MILVUS__TOKEN=...` | +| `[milvus] db_name = "default"` | `EVEROS_MILVUS__DB_NAME=default` | +| `[milvus] collection_prefix = "everos_prod"` | `EVEROS_MILVUS__COLLECTION_PREFIX=everos_prod` | diff --git a/docs/knowledge.md b/docs/knowledge.md index d3ad92dd..a7d7b177 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -48,8 +48,8 @@ Each level corresponds to a different granularity of API: ## Storage layout Every document is a self-contained directory. Markdown files are the -single source of truth; SQLite and LanceDB are derived indexes built -automatically by the cascade daemon. +single source of truth; SQLite and the configured vector/BM25 index are +derived indexes built automatically by the cascade daemon. ``` ~/.everos///knowledge/ @@ -73,16 +73,16 @@ automatically by the cascade daemon. ### Storage roles ``` -Markdown (source of truth) + SQLite (structured state) + LanceDB (vector + BM25 index) +Markdown (source of truth) + SQLite (structured state) + derived vector/BM25 index ``` | Store | What it holds | Role | |-------|---------------|------| | Markdown | Document metadata, summaries, topic content, original files | Single source of truth; human-readable and editable | | SQLite | Document rows, topic rows (with content), change queue | Structured queries, paginated lists, count aggregation | -| LanceDB | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | +| Derived index | Topic vectors, BM25 tokens, scalar fields | Search index (fully rebuildable from Markdown) | -Even if SQLite and LanceDB data is corrupted, as long as the Markdown +Even if SQLite and derived index data is corrupted, as long as the Markdown files are intact, the indexes can be fully rebuilt via the cascade daemon. ### Markdown format @@ -500,7 +500,7 @@ query ─→ embed ─→ keyword (BM25) ─┐ ``` 1. **Embed** — the query is embedded using the configured embedding provider -2. **Recall** — dual-channel retrieval from LanceDB: +2. **Recall** — dual-channel retrieval from the configured derived index: - BM25 channel: keyword matching on `summary_tokens` + `content_tokens` - ANN channel: nearest-neighbor search on the `vector` column - In `hybrid` mode, both channels run in parallel @@ -531,7 +531,7 @@ export EVEROS_KNOWLEDGE__SEARCH__RERANK_N=100 ## Cascade sync The cascade daemon watches the knowledge Markdown directory for file -changes and keeps SQLite + LanceDB in sync. +changes and keeps SQLite + the configured derived index in sync. ``` md file written @@ -539,7 +539,7 @@ md file written → worker picks up from queue (≤1s poll interval) → handler dispatched by file type: index.md → KnowledgeDocumentHandler → SQLite upsert (metadata) - N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + LanceDB upsert + N_topic.md → KnowledgeTopicHandler → tokenize + embed + SQLite + index upsert ``` The topic handler uses a SHA-256 content digest to skip unchanged files — @@ -597,7 +597,7 @@ curl .../documents?app_id=tenant_b&project_id=proj_1 # → { "documents": [], "total": 0 } ``` -Storage paths, SQLite rows, and LanceDB indexes are all scoped by +Storage paths, SQLite rows, and derived index rows are all scoped by `app_id` + `project_id`. ## End-to-end walkthrough diff --git a/docs/multimodal.md b/docs/multimodal.md index eeb18181..5c6b1a6d 100644 --- a/docs/multimodal.md +++ b/docs/multimodal.md @@ -41,7 +41,7 @@ POST /api/v2/memory/add boundary detector → extraction LLM → memory cell (MemCell) │ ▼ - markdown (truth) + SQLite (state) + LanceDB (vector + BM25) + markdown (truth) + SQLite (state) + derived vector/BM25 index │ ▼ retrievable via /search and /get like any text memory diff --git a/docs/overview.md b/docs/overview.md index 07297673..0bfda5b7 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -11,7 +11,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory - Local deployment for personal agents or small teams - Conversation, workflow, agent-trace, file-knowledge → structured memory - Hybrid retrieval (BM25 + vector + scalar filter) -- Cascade index sync (md edit → LanceDB sub-second) +- Cascade index sync (md edit → derived index) - Dual-track memory (user-track / agent-track) - Offline memory evolution (Foresight / AtomicFact / Profile / Skill), including Reflection — a consolidation strategy within the OME that @@ -30,7 +30,7 @@ Build an open-source Python memory framework where **AI agents' long-term memory ### 1. Markdown as Source of Truth ``` -delete all LanceDB / SQLite files → can rebuild from md +delete all derived index / SQLite files → can rebuild from md delete any md file → memory is gone ``` @@ -42,7 +42,7 @@ User trust comes from physical visibility — the user can `cat` / `vim` / `grep |---|---|---| | Markdown files | Truth source — entries, frontmatter | Search (grep is degraded fallback only) | | SQLite | Queue, cascade audit log, sensitive data isolation | Vector / full-text | -| LanceDB | Vector ANN + BM25 + scalar filter, single-query hybrid | Be the source of truth (loss = rebuild from md) | +| Derived index | Vector ANN + BM25 + scalar filter | Be the source of truth (loss = rebuild from md) | ### 3. Algorithm-orchestration separation diff --git a/docs/reflection.md b/docs/reflection.md index 63c09f1f..f3e920bc 100644 --- a/docs/reflection.md +++ b/docs/reflection.md @@ -147,14 +147,15 @@ narrative. ## Storage layout -Memory uses Markdown as the single source of truth; SQLite and LanceDB are -derived indexes built automatically by the cascade daemon. +Memory uses Markdown as the single source of truth; SQLite and the configured +vector/BM25 index backend are derived indexes built automatically by the +cascade daemon. | Store | What it holds | Role | |---|---|---| | Markdown | Episode bodies, merged narratives, archive markers | Single source of truth; human-readable and editable | | SQLite | Clusters and members, consolidation audit records | Structured state and queries | -| LanceDB | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | +| Derived index | Vectors + BM25 index for Episodes / atomic facts | Search (rebuildable from Markdown) | The **merged narrative** is written to the Episode daily-log Markdown; its frontmatter marks that it came from a cluster: @@ -186,7 +187,7 @@ deprecated_entries: --- ``` -> Soft-archive, not delete: even if SQLite / LanceDB are corrupted, as long +> Soft-archive, not delete: even if SQLite / the derived index are corrupted, as long > as the Markdown is intact the indexes can be fully rebuilt — and every > consolidation remains traceable back to its original content. @@ -355,5 +356,5 @@ curl -s -X POST "$BASE/memory/search" \ ## See also - [how-memory-works.md](how-memory-works.md) — Episodes and the memory extraction pipeline -- [storage_layout.md](storage_layout.md) — Markdown + SQLite + LanceDB stack +- [storage_layout.md](storage_layout.md) — Markdown + SQLite + derived index stack - [api.md](api.md) — full HTTP API reference diff --git a/docs/storage_layout.md b/docs/storage_layout.md index 86044664..3d90c775 100644 --- a/docs/storage_layout.md +++ b/docs/storage_layout.md @@ -3,8 +3,9 @@ How `everos` lays out a memory-root on disk: directory tree, file naming, frontmatter chassis, and entry-id encoding. -The contents are the **source of truth**; SQLite and LanceDB are -derived indexes that can be rebuilt from markdown alone. +The contents are the **source of truth**; SQLite and the configured +vector/BM25 index backend are derived indexes that can be rebuilt from +markdown alone. ## 1. Memory-root tree @@ -49,8 +50,8 @@ the frontmatter (see [§3](#3-frontmatter-chassis-yaml)). │ │ ├── ome.db Offline Memory Engine state │ │ ├── ome.aps.db APScheduler jobstore (split to avoid lock contention) │ │ └── ome.db.lock OME single-engine guard (portalocker) -│ └── lancedb/ -│ └── .lance/ one directory per LanceDB table +│ ├── lancedb/ +│ │ └── .lance/ default derived index backend │ ├── ome.toml user-editable OME strategy overrides (hot-reloaded) └── .tmp/ staging dir for batch / multi-step writes @@ -168,9 +169,9 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers > **File-level seq, not global**: the same `ep_20260601_00000001` may > appear across two different `user_id`s (each user has its own daily file). > Cross-table joins must therefore key on **`(scope_id, entry_id)`** -> rather than `entry_id` alone — see SQLite/LanceDB tables that follow. +> rather than `entry_id` alone — see the derived index tables that follow. -## 5. SQLite + LanceDB derived indexes +## 5. SQLite + derived indexes ``` .index/ @@ -178,9 +179,8 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers │ └── system.db state / audit / cascade queue + buffer / LSN │ (system tables: md_change_state, memcell, │ unprocessed_buffer, conversation_status, cluster) -└── lancedb/ - └── .lance/ one Arrow table per business kind — the per-kind - rows (text / vector / tokens / metadata) live here +├── lancedb/ +│ └── .lance/ default derived index backend ``` - **SQLite** ([`infra/persistence/sqlite/tables/`](../src/everos/infra/persistence/sqlite/tables/)) @@ -190,13 +190,14 @@ Implementation: [`core/persistence/markdown/entries.py`](../src/everos/core/pers per-kind business rows. `reflection_report` is the audit trail for Reflection merges (cluster_id, mode, source_members, merged_entry_id, status). -- **LanceDB** ([`infra/persistence/lancedb/tables/`](../src/everos/infra/persistence/lancedb/tables/)) - holds the per-kind business rows, keyed `_` (so - cross-table joins use `(owner_id, entry_id)`); each table's `Vector(N)` - dimension matches the embedding model output. - -Episode and AtomicFact LanceDB tables carry a `deprecated_by: str | None` -column. When an episode is superseded by a Reflection merge, +- The **derived index backend** holds the per-kind business rows, keyed + `_` (so cross-table joins use `(owner_id, entry_id)`). + LanceDB is the default backend under `.index/lancedb/`; Milvus can be enabled + as the same rebuildable index backend and lives outside the memory root in a + configured Milvus Server or Zilliz Cloud deployment. + +Episode and AtomicFact index rows carry a `deprecated_by: str | None` column. +When an episode is superseded by a Reflection merge, `deprecated_by` is set to the merged episode's entry_id. Search filters automatically exclude rows where `deprecated_by IS NOT NULL`. @@ -226,5 +227,5 @@ this primitive is **schema-agnostic** — field-level semantics - Code: - [`core/persistence/memory_root.py`](../src/everos/core/persistence/memory_root.py) - [`core/persistence/markdown/`](../src/everos/core/persistence/markdown/) - - [`infra/persistence/{markdown,sqlite,lancedb}/`](../src/everos/infra/persistence/) - - [`memory/cascade/`](../src/everos/memory/cascade/) (md → LanceDB sync) + - [`infra/persistence/{markdown,sqlite,lancedb,milvus,index}/`](../src/everos/infra/persistence/) + - [`memory/cascade/`](../src/everos/memory/cascade/) (md → derived index sync) diff --git a/pyproject.toml b/pyproject.toml index 991d9c91..7006dcd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ otel = [ "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", ] +milvus = ["pymilvus>=3.0.0"] [project.urls] Homepage = "https://evermind.ai" @@ -249,6 +250,7 @@ ignore_imports = [ "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.tables", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.repos", "everos.infra.persistence.lancedb -> everos.infra.persistence.lancedb.lancedb_manager", + "everos.infra.persistence.index -> everos.infra.persistence.lancedb.predicate", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.mds", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.writers", "everos.infra.persistence.markdown -> everos.infra.persistence.markdown.readers", @@ -280,4 +282,7 @@ dev = [ # [otel] stack is always present in the dev / CI environment. "opentelemetry-sdk>=1.27.0", "opentelemetry-exporter-otlp-proto-http>=1.27.0", + # Milvus tests must run in every dev/CI environment. The client package + # intentionally excludes the embedded milvus-lite extra. + "pymilvus>=3.0.0", ] diff --git a/src/everos/README.md b/src/everos/README.md index 8cc1cef9..2be6caf2 100644 --- a/src/everos/README.md +++ b/src/everos/README.md @@ -10,7 +10,7 @@ everos/ ├── entrypoints/ Presentation: cli + api ├── service/ Application: use case orchestration ├── memory/ Domain: extract + search + cascade + prompt_slots + models -├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb} +├── infra/ Infrastructure: persistence/{markdown, sqlite, lancedb, milvus, index} ├── component/ Cross-cutting providers: llm / embedding / config / utils ├── core/ Runtime base: observability / lifespan / context └── config/ Data: Settings + default.toml + prompt_slots templates diff --git a/src/everos/config/__init__.py b/src/everos/config/__init__.py index adbd1e35..6ce3d968 100644 --- a/src/everos/config/__init__.py +++ b/src/everos/config/__init__.py @@ -2,7 +2,8 @@ Public API: from everos.config import ( - Settings, MemorySettings, SqliteSettings, LanceDBSettings, + Settings, MemorySettings, SqliteSettings, IndexSettings, LanceDBSettings, + MilvusSettings, LLMSettings, EmbeddingSettings, RerankSettings, BoundaryDetectionSettings, CascadeSettings, load_settings, resolve_root, @@ -15,9 +16,11 @@ from .settings import BoundaryDetectionSettings as BoundaryDetectionSettings from .settings import CascadeSettings as CascadeSettings from .settings import EmbeddingSettings as EmbeddingSettings +from .settings import IndexSettings as IndexSettings from .settings import LanceDBSettings as LanceDBSettings from .settings import LLMSettings as LLMSettings from .settings import MemorySettings as MemorySettings +from .settings import MilvusSettings as MilvusSettings from .settings import MultimodalSettings as MultimodalSettings from .settings import RerankSettings as RerankSettings from .settings import Settings as Settings @@ -28,9 +31,11 @@ __all__ = [ "BoundaryDetectionSettings", "EmbeddingSettings", + "IndexSettings", "LLMSettings", "LanceDBSettings", "MemorySettings", + "MilvusSettings", "MultimodalSettings", "RerankSettings", "Settings", diff --git a/src/everos/config/default.toml b/src/everos/config/default.toml index e2137f84..38d22682 100644 --- a/src/everos/config/default.toml +++ b/src/everos/config/default.toml @@ -50,6 +50,24 @@ cache_size_kb = 2048 # Uncomment to override: # read_consistency_seconds = 5.0 +[index] +# Rebuildable vector/BM25 index backend. Markdown remains the source of truth +# and SQLite remains the system state store. +# Override via EVEROS_INDEX__BACKEND. +backend = "lancedb" + +[milvus] +# Optional remote derived index backend for Milvus Server or Zilliz Cloud. +# `uri` is required when [index] backend = "milvus". Embedded Milvus Lite and +# local database paths are not supported. Use a unique db_name or +# collection_prefix when multiple EverOS deployments share a cluster. +# Override via EVEROS_MILVUS__URI, EVEROS_MILVUS__TOKEN, EVEROS_MILVUS__DB_NAME. +uri = "" +token = "" +db_name = "" +consistency_level = "Session" +collection_prefix = "everos" + [llm] # Provider-agnostic OpenAI-protocol client config. Override via env: # EVEROS_LLM__MODEL, EVEROS_LLM__API_KEY, EVEROS_LLM__BASE_URL diff --git a/src/everos/config/settings.py b/src/everos/config/settings.py index 50da9947..1e910ac8 100644 --- a/src/everos/config/settings.py +++ b/src/everos/config/settings.py @@ -23,6 +23,7 @@ from __future__ import annotations import os +import re from functools import cache from pathlib import Path from typing import Literal @@ -375,6 +376,37 @@ class CascadeSettings(BaseModel): optimize_rebuild_interval_seconds: float = 12 * 60 * 60.0 +class IndexSettings(BaseModel): + """Derived-index backend selection.""" + + backend: Literal["lancedb", "milvus"] = "lancedb" + + +class MilvusSettings(BaseModel): + """Remote Milvus Server or Zilliz Cloud connection settings. + + Embedded Milvus Lite is intentionally unsupported. Deployments that select + the Milvus backend must configure an external endpoint and should use a + unique database or collection prefix when sharing a cluster. + """ + + uri: str = "" + token: SecretStr = SecretStr("") + db_name: str = "" + consistency_level: Literal["Strong", "Bounded", "Session", "Eventually"] = "Session" + collection_prefix: str = Field(default="everos", min_length=1) + + @field_validator("collection_prefix") + @classmethod + def _validate_collection_prefix(cls, value: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value): + raise ValueError( + "collection_prefix must start with a letter or underscore and " + "contain only letters, digits, and underscores" + ) + return value + + class KnowledgeSearchSettings(BaseModel): """``[knowledge.search]`` — retrieval tuning for the knowledge module.""" @@ -448,6 +480,8 @@ class Settings(BaseSettings): api: ApiSettings = ApiSettings() sqlite: SqliteSettings = SqliteSettings() lancedb: LanceDBSettings = LanceDBSettings() + index: IndexSettings = IndexSettings() + milvus: MilvusSettings = MilvusSettings() llm: LLMSettings = LLMSettings() embedding: EmbeddingSettings = EmbeddingSettings() rerank: RerankSettings = RerankSettings() diff --git a/src/everos/entrypoints/api/lifespans/lancedb.py b/src/everos/entrypoints/api/lifespans/lancedb.py index d20e05b6..56994f56 100644 --- a/src/everos/entrypoints/api/lifespans/lancedb.py +++ b/src/everos/entrypoints/api/lifespans/lancedb.py @@ -1,4 +1,4 @@ -"""LanceDB lifespan provider (HTTP API entrypoint). +"""Derived-index lifespan provider (HTTP API entrypoint). Startup: Open the connection via ``get_connection`` (lazy, idempotent). @@ -31,13 +31,18 @@ from everos.core.lifespan import LifespanProvider from everos.core.observability.logging import get_logger +from everos.infra.persistence.index import ( + active_backend, +) +from everos.infra.persistence.index import ( + shutdown as shutdown_index, +) +from everos.infra.persistence.index import ( + startup as startup_index, +) from everos.infra.persistence.lancedb import ( BUSINESS_SCHEMAS_WITH_VECTOR, - dispose_connection, - ensure_business_indexes, - get_connection, get_table, - verify_business_schemas, ) logger = get_logger(__name__) @@ -83,7 +88,7 @@ async def _log_unbackfilled_hint() -> None: class LanceDBLifespanProvider(LifespanProvider): - """Manage the LanceDB connection + table cache for the app lifecycle. + """Manage the configured derived index for the app lifecycle. Startup runs four steps: @@ -100,12 +105,12 @@ def __init__(self, order: int = 11) -> None: super().__init__(name="lancedb", order=order) async def startup(self, app: FastAPI) -> Any: - conn = await get_connection() - await verify_business_schemas() - await ensure_business_indexes() - await _log_unbackfilled_hint() - logger.info("lancedb_ready", uri=conn.uri) + backend = active_backend() + conn = await startup_index() + if backend == "lancedb": + await _log_unbackfilled_hint() + logger.info("derived_index_ready", backend=backend) return conn async def shutdown(self, app: FastAPI) -> None: - await dispose_connection() + await shutdown_index() diff --git a/src/everos/entrypoints/cli/commands/cascade.py b/src/everos/entrypoints/cli/commands/cascade.py index 8710f947..d39f4694 100644 --- a/src/everos/entrypoints/cli/commands/cascade.py +++ b/src/everos/entrypoints/cli/commands/cascade.py @@ -15,7 +15,7 @@ vectors, build clusters, extract skills. See :func:`everos.entrypoints.cli.commands._backfill_cmd.run_backfill` for the phase orchestration. -- ``cascade rebuild`` — drop every business LanceDB table and re-index +- ``cascade rebuild`` — drop every business derived-index table and re-index all md from scratch. Recovery for a drifted / corrupt index; safe because md is the source of truth and un-extracted buffered messages are preserved. Skips the schema-verify guard (which the drift would @@ -46,13 +46,16 @@ from everos.core.persistence import MemoryRoot from everos.entrypoints.cli._log_setup import configure_cli_logging from everos.entrypoints.cli.commands._backfill_cmd import run_backfill -from everos.infra.persistence.lancedb import ( - dispose_connection, +from everos.infra.persistence.index import ( + active_backend, + connect, drop_business_tables, ensure_business_indexes, - get_connection, verify_business_schemas, ) +from everos.infra.persistence.index import ( + shutdown as shutdown_index, +) from everos.infra.persistence.sqlite import ( dispose_engine, get_engine, @@ -68,7 +71,7 @@ app = typer.Typer( name="cascade", - help="Inspect and operate the md → LanceDB sync queue", + help="Inspect and operate the markdown → derived-index sync queue", no_args_is_help=True, ) @@ -138,7 +141,7 @@ def _apply_verbose_logging(verbose: bool | None) -> None: async def _runtime( # type: ignore[no-untyped-def] *, verify: bool = True, ensure: bool = True ): - """Stand up sqlite + lancedb the same way the API lifespan would. + """Stand up SQLite + the configured index the way the API lifespan would. The CLI uses the same lazy, process-wide singletons the API lifespan does. They are **per-process**: a running daemon has its own @@ -163,15 +166,21 @@ async def _runtime( # type: ignore[no-untyped-def] engine = get_engine() async with engine.begin() as conn: await conn.run_sync(SQLModel.metadata.create_all) - await get_connection() - if verify: - await verify_business_schemas() - if ensure: - await ensure_business_indexes() + await connect() + if active_backend() == "milvus": + if ensure: + await ensure_business_indexes() + if verify: + await verify_business_schemas() + else: + if verify: + await verify_business_schemas() + if ensure: + await ensure_business_indexes() try: yield finally: - await dispose_connection() + await shutdown_index() await dispose_engine() @@ -451,7 +460,7 @@ def rebuild( typer.Option("--yes", "-y", help="Skip the confirmation prompt."), ] = False, ) -> None: - """Rebuild the LanceDB index from markdown (recover from schema drift). + """Rebuild the configured derived index from markdown. **Stop the ``everos server`` first** — this is the one cascade command that is not safe alongside a live daemon. It drops and recreates the @@ -459,13 +468,13 @@ def rebuild( the dropped dataset; the command refuses to start while a server holds the OME lock. - Drops every business LanceDB table and re-indexes all md from + Drops every business table or collection and re-indexes all markdown from scratch. Markdown is the source of truth, so no memory content is lost, and this is the safe recovery from a drifted / corrupt index (e.g. the ``verify_business_schemas`` startup failure): - - unlike ``rm -rf ~/.everos/.index/lancedb``, it re-populates - already-indexed entries (that command leaves the cascade queue + - unlike deleting ``~/.everos/.index/lancedb`` on the default backend, it + re-populates already-indexed entries (that command leaves the cascade queue marked ``done``, so nothing re-indexes and the index comes back empty); - unlike ``rm -rf ~/.everos/.index``, it preserves SQLite state that @@ -476,7 +485,7 @@ def rebuild( typer.echo( "error: a server (or another exclusive CLI phase) is running on " "this memory root.\n" - " cascade rebuild drops and recreates the LanceDB tables; a live " + " cascade rebuild drops and recreates the derived index; a live " "daemon holds cached\n" " table handles and would keep writing to the dropped dataset. " "Stop `everos server`\n" @@ -486,7 +495,7 @@ def rebuild( raise typer.Exit(code=3) if not yes: typer.confirm( - "Drop all LanceDB business tables and re-index from markdown? " + "Drop all derived-index business data and re-index from markdown? " "(requires the server to be stopped)", abort=True, ) @@ -507,7 +516,7 @@ async def _run() -> None: typer.echo(f"reset {cleared} cascade queue row(s)") dropped = await drop_business_tables() typer.echo( - f"dropped {len(dropped)} LanceDB table(s): " + f"dropped {len(dropped)} derived-index table(s): " f"{', '.join(dropped) or '(none)'}" ) # Recreate the tables (current schema) + FTS indexes. diff --git a/src/everos/infra/persistence/index/__init__.py b/src/everos/infra/persistence/index/__init__.py new file mode 100644 index 00000000..dfe2bcab --- /dev/null +++ b/src/everos/infra/persistence/index/__init__.py @@ -0,0 +1,395 @@ +"""Backend-neutral derived index persistence facade. + +Markdown is the source of truth and SQLite stores system state. This package +selects the rebuildable vector/BM25 index backend used by cascade, search, and +get. The default backend remains LanceDB; Milvus is opt-in through +``[index] backend = "milvus"``. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from lancedb.query import BooleanQuery, FullTextQuery, MatchQuery + +try: + from lancedb.query import Occur +except ImportError: # pragma: no cover + from lancedb._lancedb import Occur # type: ignore[attr-defined,no-redef] + +from everos.config import load_settings +from everos.infra.persistence import lancedb as _lancedb +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + ParentType, + UserProfile, +) +from everos.infra.persistence.lancedb.predicate import render_predicate as _render_lance + +from .predicate import ( + Predicate, + all_of, + any_of, + contains, + eq, + gt, + gte, + is_null, + lt, + lte, + ne, + one_of, +) +from .schema import schema_for + + +def active_backend() -> str: + """Return the configured derived index backend name.""" + return load_settings().index.backend + + +async def startup() -> Any: + """Initialise the configured derived index backend.""" + if active_backend() == "milvus": + milvus = _milvus() + await milvus.get_client() + await milvus.ensure_business_indexes() + return "milvus" + conn = await _lancedb.get_connection() + await _lancedb.verify_business_schemas() + await _lancedb.ensure_business_indexes() + return conn + + +async def connect() -> Any: + """Open the active backend without creating or verifying indexes.""" + if active_backend() == "milvus": + return await _milvus().get_client() + return await _lancedb.get_connection() + + +async def shutdown() -> None: + """Dispose the configured derived index backend.""" + if active_backend() == "milvus": + await _milvus().dispose_connection() + else: + await _lancedb.dispose_connection() + + +async def ensure_business_indexes() -> None: + if active_backend() == "milvus": + await _milvus().ensure_business_indexes() + else: + await _lancedb.ensure_business_indexes() + + +async def verify_business_schemas() -> None: + if active_backend() == "milvus": + await _milvus().verify_business_schemas() + else: + await _lancedb.verify_business_schemas() + + +async def drop_business_tables() -> list[str]: + """Drop all business indexes for the active backend.""" + if active_backend() == "milvus": + return await _milvus().drop_business_tables() + return await _lancedb.drop_business_tables() + + +class _LanceIndexRepoAdapter: + """Add backend-neutral recall helpers to an existing LanceDB repo.""" + + def __init__(self, repo: Any, schema: type[Any]) -> None: + self._repo = repo + self.schema = schema + self.index_schema = schema_for(schema) + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo, name) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + fields = list(columns or self.index_schema.bm25_fields) + if not query_terms or not fields: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + expr = _lance_expr(where) + best: dict[str, dict[str, Any]] = {} + for field in fields: + query = _build_or_query(query_terms, field) + rows = ( + await table.query() + .nearest_to_text(query) + .where(expr) + .limit(limit) + .to_list() + ) + for row in rows: + rid = row.get("id") + if not isinstance(rid, str): + continue + score = float(row.get("_score", 0.0)) + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + shaped = dict(row) + shaped["_score"] = score + best[rid] = shaped + return sorted( + best.values(), key=lambda row: float(row.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + if not vector: + return [] + table = await _lancedb.get_table(self.schema.TABLE_NAME, self.schema) + return ( + await table.query() + .nearest_to(list(vector)) + .column(vector_field) + .distance_type("cosine") + .where(_lance_expr(where)) + .limit(limit) + .to_list() + ) + + +class _IndexRepoRouter: + """Route repo calls to the configured derived index backend.""" + + def __init__( + self, lance_repo: Any, milvus_repo_name: str, schema: type[Any] + ) -> None: + self._lance = _LanceIndexRepoAdapter(lance_repo, schema) + self._milvus_repo_name = milvus_repo_name + self.schema = schema + self.index_schema = schema_for(schema) + + @property + def table_name(self) -> str: + return self.index_schema.table_name + + def _repo(self) -> Any: + if active_backend() == "milvus": + return getattr(_milvus(), self._milvus_repo_name) + return self._lance + + def __getattr__(self, name: str) -> Any: + return getattr(self._repo(), name) + + async def add(self, records: Sequence[Any]) -> None: + await self._repo().add(records) + + async def upsert(self, records: Sequence[Any], *, by: str = "id") -> None: + await self._repo().upsert(records, by=by) + + async def count(self) -> int: + return await self._repo().count() + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> Any: + return await self._repo().get_by_id(id_value, id_field=id_field) + + async def find_where(self, where: Any, *, limit: int = 100) -> list[Any]: + return await self._repo().find_where(_where_for_backend(where), limit=limit) + + async def find_one_where(self, where: Any) -> Any: + return await self._repo().find_one_where(_where_for_backend(where)) + + async def find_where_paginated( + self, + where: Any, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[Any], int]: + return await self._repo().find_where_paginated( + _where_for_backend(where), + sort_by=sort_by, + descending=descending, + page=page, + page_size=page_size, + max_fetch=max_fetch, + ) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Any = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + return await self._repo().search( + vector=vector, where=_where_for_backend(where), limit=limit + ) + + async def update(self, updates: dict[str, Any], *, where: Any) -> None: + await self._repo().update(updates, where=_where_for_backend(where)) + + async def delete(self, predicate: Any) -> None: + await self._repo().delete(_where_for_backend(predicate)) + + async def delete_by_md_path(self, md_path: str) -> int: + return await self._repo().delete_by_md_path(md_path) + + async def optimize(self, **kwargs: Any) -> None: + await self._repo().optimize(**kwargs) + + async def rebuild_indexes(self) -> None: + await self._repo().rebuild_indexes() + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Any, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + return await self._repo().sparse_search( + query_terms, + _where_for_backend(where), + columns=columns, + limit=limit, + ) + + async def dense_search( + self, + vector: Sequence[float], + where: Any, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + return await self._repo().dense_search( + vector, + _where_for_backend(where), + limit=limit, + vector_field=vector_field, + ) + + +def _where_for_backend(where: Any) -> Any: + if where is None: + return None if active_backend() == "milvus" else "" + if not isinstance(where, Predicate): + raise TypeError( + "derived-index repository predicates must use the neutral " + f"Predicate AST, got {type(where).__name__}" + ) + if active_backend() == "milvus": + return where + return _render_lance(where) + + +def _lance_expr(where: Any) -> str: + if isinstance(where, Predicate): + return _render_lance(where) + if isinstance(where, str): + return where + raise TypeError(f"unsupported LanceDB predicate: {type(where).__name__}") + + +def _build_or_query(tokens: Sequence[str], column: str) -> FullTextQuery: + clean = [token for token in tokens if token] + if not clean: + return MatchQuery("", column=column) + if len(clean) == 1: + return MatchQuery(clean[0], column=column) + return BooleanQuery( + [(Occur.SHOULD, MatchQuery(token, column=column)) for token in clean] + ) + + +def _milvus() -> Any: + from everos.infra.persistence import milvus + + return milvus + + +episode_repo = _IndexRepoRouter(_lancedb.episode_repo, "episode_repo", Episode) +atomic_fact_repo = _IndexRepoRouter( + _lancedb.atomic_fact_repo, "atomic_fact_repo", AtomicFact +) +foresight_repo = _IndexRepoRouter(_lancedb.foresight_repo, "foresight_repo", Foresight) +agent_case_repo = _IndexRepoRouter( + _lancedb.agent_case_repo, "agent_case_repo", AgentCase +) +agent_skill_repo = _IndexRepoRouter( + _lancedb.agent_skill_repo, "agent_skill_repo", AgentSkill +) +user_profile_repo = _IndexRepoRouter( + _lancedb.user_profile_repo, "user_profile_repo", UserProfile +) +knowledge_topic_repo = _IndexRepoRouter( + _lancedb.knowledge_topic_repo, "knowledge_topic_repo", KnowledgeTopic +) + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "AgentCase", + "AgentSkill", + "AtomicFact", + "Episode", + "Foresight", + "KnowledgeTopic", + "ParentType", + "Predicate", + "UserProfile", + "active_backend", + "agent_case_repo", + "agent_skill_repo", + "all_of", + "any_of", + "atomic_fact_repo", + "connect", + "contains", + "drop_business_tables", + "ensure_business_indexes", + "episode_repo", + "eq", + "foresight_repo", + "gt", + "gte", + "is_null", + "knowledge_topic_repo", + "lt", + "lte", + "ne", + "one_of", + "shutdown", + "startup", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/index/predicate.py b/src/everos/infra/persistence/index/predicate.py new file mode 100644 index 00000000..4892356e --- /dev/null +++ b/src/everos/infra/persistence/index/predicate.py @@ -0,0 +1,45 @@ +"""Compatibility re-exports for the shared derived-index predicate AST.""" + +from everos.infra.persistence.predicate import All as All +from everos.infra.persistence.predicate import AnyOf as AnyOf +from everos.infra.persistence.predicate import Comparison as Comparison +from everos.infra.persistence.predicate import Contains as Contains +from everos.infra.persistence.predicate import In as In +from everos.infra.persistence.predicate import IsNull as IsNull +from everos.infra.persistence.predicate import Predicate as Predicate +from everos.infra.persistence.predicate import Scalar as Scalar +from everos.infra.persistence.predicate import all_of as all_of +from everos.infra.persistence.predicate import any_of as any_of +from everos.infra.persistence.predicate import compare as compare +from everos.infra.persistence.predicate import contains as contains +from everos.infra.persistence.predicate import eq as eq +from everos.infra.persistence.predicate import gt as gt +from everos.infra.persistence.predicate import gte as gte +from everos.infra.persistence.predicate import is_null as is_null +from everos.infra.persistence.predicate import lt as lt +from everos.infra.persistence.predicate import lte as lte +from everos.infra.persistence.predicate import ne as ne +from everos.infra.persistence.predicate import one_of as one_of + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/infra/persistence/index/schema.py b/src/everos/infra/persistence/index/schema.py new file mode 100644 index 00000000..dd9294ba --- /dev/null +++ b/src/everos/infra/persistence/index/schema.py @@ -0,0 +1,341 @@ +"""Backend-neutral schemas for rebuildable business indexes. + +Each table has one explicit field description consumed by every backend +adapter. The model-field parity check fails immediately when a domain field is +added without a storage decision, avoiding fallback coercions and silently +divergent schemas. +""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from enum import StrEnum +from functools import cache +from typing import Any, get_args, get_origin + +_DEFAULT_STRING_LENGTH = 65_535 +_ID_LENGTH = 512 +_ARRAY_CAPACITY = 256 +_VECTOR_DIMENSION = 1024 + + +class IndexFieldKind(StrEnum): + STRING = "string" + STRING_ARRAY = "string_array" + FLOAT = "float" + INTEGER = "integer" + DATETIME = "datetime" + DENSE_VECTOR = "dense_vector" + + +@dataclass(frozen=True) +class IndexField: + name: str + kind: IndexFieldKind + nullable: bool = False + primary: bool = False + max_length: int | None = None + max_capacity: int | None = None + dimension: int | None = None + + +@dataclass(frozen=True) +class IndexSchema: + table_name: str + model: type[Any] + fields: tuple[IndexField, ...] + bm25_fields: tuple[str, ...] + + def field(self, name: str) -> IndexField: + for field in self.fields: + if field.name == name: + return field + raise KeyError(name) + + @property + def vector_fields(self) -> tuple[IndexField, ...]: + return tuple( + field for field in self.fields if field.kind is IndexFieldKind.DENSE_VECTOR + ) + + @property + def datetime_fields(self) -> frozenset[str]: + return frozenset( + field.name for field in self.fields if field.kind is IndexFieldKind.DATETIME + ) + + +def _s( + name: str, + *, + nullable: bool = False, + primary: bool = False, + max_length: int = _DEFAULT_STRING_LENGTH, +) -> IndexField: + return IndexField( + name, + IndexFieldKind.STRING, + nullable=nullable, + primary=primary, + max_length=max_length, + ) + + +def _id(name: str = "id") -> IndexField: + return _s(name, primary=True, max_length=_ID_LENGTH) + + +def _a(name: str, *, nullable: bool = False) -> IndexField: + return IndexField( + name, + IndexFieldKind.STRING_ARRAY, + nullable=nullable, + max_length=_ID_LENGTH, + max_capacity=_ARRAY_CAPACITY, + ) + + +def _f(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.FLOAT, nullable=nullable) + + +def _i(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.INTEGER, nullable=nullable) + + +def _d(name: str, *, nullable: bool = False) -> IndexField: + return IndexField(name, IndexFieldKind.DATETIME, nullable=nullable) + + +def _v(name: str, *, nullable: bool = True) -> IndexField: + return IndexField( + name, + IndexFieldKind.DENSE_VECTOR, + nullable=nullable, + dimension=_VECTOR_DIMENSION, + ) + + +_FIELDS: dict[str, tuple[IndexField, ...]] = { + "episode": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("subject", nullable=True), + _s("summary", nullable=True), + _s("episode"), + _s("episode_tokens"), + _s("md_path"), + _s("content_sha256"), + _s("deprecated_by", nullable=True), + _v("vector"), + _v("subject_vector"), + _d("created_at"), + _d("updated_at"), + ), + "atomic_fact": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("fact"), + _s("fact_tokens"), + _s("md_path"), + _s("content_sha256"), + _s("deprecated_by", nullable=True), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "foresight": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id", nullable=True), + _d("timestamp"), + _d("start_time", nullable=True), + _d("end_time", nullable=True), + _i("duration_days", nullable=True), + _s("parent_type"), + _s("parent_id"), + _a("sender_ids"), + _s("foresight"), + _s("foresight_tokens"), + _s("evidence", nullable=True), + _s("evidence_tokens", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "agent_case": ( + _id(), + _s("entry_id"), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("session_id"), + _d("timestamp"), + _s("parent_type"), + _s("parent_id"), + _f("quality_score"), + _s("task_intent"), + _s("task_intent_tokens"), + _s("approach"), + _s("approach_tokens"), + _s("key_insight", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "agent_skill": ( + _id(), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("name"), + _s("description"), + _s("description_tokens"), + _s("content"), + _s("content_tokens"), + _f("confidence"), + _f("maturity_score"), + _a("source_case_ids"), + _s("cluster_id", nullable=True), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), + "user_profile": ( + _id(), + _s("owner_id"), + _s("owner_type"), + _s("app_id"), + _s("project_id"), + _s("summary"), + _s("explicit_info_json"), + _s("implicit_traits_json"), + _i("profile_timestamp_ms"), + _s("md_path"), + _s("content_sha256"), + _d("created_at"), + _d("updated_at"), + ), + "knowledge_topic": ( + _id(), + _s("doc_id"), + _s("category_id"), + _s("app_id"), + _s("project_id"), + _s("topic_name"), + _s("topic_path"), + _i("depth"), + _s("parent_node_id"), + _s("summary"), + _s("summary_tokens"), + _s("content_tokens"), + _a("content_labels"), + _s("md_path"), + _s("content_sha256"), + _v("vector"), + _d("created_at"), + _d("updated_at"), + ), +} + + +@cache +def schema_for(model: type[Any]) -> IndexSchema: + """Return and validate the explicit neutral schema for a model.""" + table_name = model.TABLE_NAME + try: + fields = _FIELDS[table_name] + except KeyError as exc: + raise ValueError(f"no derived-index schema for {table_name!r}") from exc + + declared = {field.name for field in fields} + actual = set(model.model_fields) + if declared != actual: + raise ValueError( + f"derived-index schema drift for {table_name!r}: " + f"missing={sorted(actual - declared)}, stale={sorted(declared - actual)}" + ) + for field in fields: + _validate_model_field(table_name, field, model.model_fields[field.name]) + bm25_fields = tuple(model.BM25_FIELDS) + unknown_bm25 = set(bm25_fields) - declared + if unknown_bm25: + raise ValueError( + f"derived-index schema {table_name!r} has unknown BM25 fields: " + f"{sorted(unknown_bm25)}" + ) + return IndexSchema(table_name, model, fields, bm25_fields) + + +def _validate_model_field(table_name: str, field: IndexField, model_field: Any) -> None: + annotation = model_field.annotation + args = get_args(annotation) + optional = type(None) in args + candidates = ( + tuple(arg for arg in args if arg is not type(None)) + if optional + else (annotation,) + ) + if optional != field.nullable: + raise ValueError( + f"derived-index schema {table_name}.{field.name} nullable drift: " + f"model={optional}, schema={field.nullable}" + ) + + valid = False + if field.kind is IndexFieldKind.STRING: + valid = candidates == (str,) + elif field.kind is IndexFieldKind.STRING_ARRAY: + valid = len(candidates) == 1 and ( + get_origin(candidates[0]) is list and get_args(candidates[0]) == (str,) + ) + elif field.kind is IndexFieldKind.FLOAT: + valid = candidates == (float,) + elif field.kind is IndexFieldKind.INTEGER: + valid = candidates == (int,) + elif field.kind is IndexFieldKind.DATETIME: + valid = candidates == (dt.datetime,) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + dimension = getattr(candidates[0], "dim", None) if candidates else None + if callable(dimension): + dimension = dimension() + valid = len(candidates) == 1 and dimension == field.dimension + if not valid: + raise ValueError( + f"derived-index schema {table_name}.{field.name} type drift: " + f"model={annotation!r}, schema={field.kind.value}" + ) + + +__all__ = ["IndexField", "IndexFieldKind", "IndexSchema", "schema_for"] diff --git a/src/everos/infra/persistence/lancedb/predicate.py b/src/everos/infra/persistence/lancedb/predicate.py new file mode 100644 index 00000000..8a429d57 --- /dev/null +++ b/src/everos/infra/persistence/lancedb/predicate.py @@ -0,0 +1,83 @@ +"""Render backend-neutral predicates as LanceDB DataFusion expressions.""" + +from __future__ import annotations + +import datetime as dt +import re +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_iso_format +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate(predicate: Predicate | None) -> str: + """Render a predicate with LanceDB escaping and timestamp literals.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field)} {_OPERATORS[predicate.operator]} " + f"{_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field)} IN ({values})" + if isinstance(predicate, Contains): + return f"array_has({_field(predicate.field)}, {_literal(predicate.value)})" + if isinstance(predicate, IsNull): + return f"{_field(predicate.field)} IS NULL" + if isinstance(predicate, All): + return _render_group(predicate.children, "AND") + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "OR") + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group(children: tuple[Predicate, ...], operator: str) -> str: + rendered = [render_predicate(child) for child in children] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return f"'{value.replace(chr(39), chr(39) * 2)}'" + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return f"TIMESTAMP '{to_iso_format(aware)}'" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + return str(value) + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/__init__.py b/src/everos/infra/persistence/milvus/__init__.py new file mode 100644 index 00000000..910f8e64 --- /dev/null +++ b/src/everos/infra/persistence/milvus/__init__.py @@ -0,0 +1,85 @@ +"""Milvus derived index backend. + +This package mirrors the LanceDB business index surface but stores rows in +Milvus collections. It is selected through ``Settings.index.backend`` and is +normally reached through :mod:`everos.infra.persistence.index`. +""" + +from __future__ import annotations + +import asyncio + +from everos.core.observability.logging import get_logger + +from .milvus_manager import MilvusConfigurationError as MilvusConfigurationError +from .milvus_manager import MilvusSchemaMismatchError as MilvusSchemaMismatchError +from .milvus_manager import dispose_connection as dispose_connection +from .milvus_manager import get_client as get_client +from .repos import ALL_REPOS as ALL_REPOS +from .repos import agent_case_repo as agent_case_repo +from .repos import agent_skill_repo as agent_skill_repo +from .repos import atomic_fact_repo as atomic_fact_repo +from .repos import episode_repo as episode_repo +from .repos import foresight_repo as foresight_repo +from .repos import knowledge_topic_repo as knowledge_topic_repo +from .repos import user_profile_repo as user_profile_repo +from .repository import MilvusRepoBase as MilvusRepoBase +from .repository import MilvusValueLimitError as MilvusValueLimitError + +logger = get_logger(__name__) + + +async def ensure_business_indexes() -> None: + """Create or verify every EverOS Milvus collection.""" + for repo in ALL_REPOS: + await repo.ensure_collection() + + +async def verify_business_schemas() -> None: + """Fail loud if existing Milvus collections drift from EverOS schemas.""" + for repo in ALL_REPOS: + await repo.verify_collection() + + +async def drop_business_tables() -> list[str]: + """Drop every configured Milvus collection and return their names.""" + client = await get_client() + dropped: list[str] = [] + for repo in ALL_REPOS: + name = repo.collection_name + if await asyncio.to_thread(client.has_collection, name): + try: + await asyncio.to_thread(client.drop_collection, name) + except Exception: + # Zilliz Serverless can complete the drop server-side while + # its gateway returns DEADLINE_EXCEEDED. Confirm state before + # turning an already-successful rebuild/cleanup into failure. + if await asyncio.to_thread(client.has_collection, name): + raise + logger.warning( + "milvus_collection_drop_confirmed_after_client_error", + collection=name, + ) + dropped.append(name) + MilvusRepoBase._reset_collection_cache() + return dropped + + +__all__ = [ + "ALL_REPOS", + "MilvusConfigurationError", + "MilvusSchemaMismatchError", + "MilvusValueLimitError", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "dispose_connection", + "drop_business_tables", + "ensure_business_indexes", + "episode_repo", + "foresight_repo", + "get_client", + "knowledge_topic_repo", + "user_profile_repo", + "verify_business_schemas", +] diff --git a/src/everos/infra/persistence/milvus/milvus_manager.py b/src/everos/infra/persistence/milvus/milvus_manager.py new file mode 100644 index 00000000..05af230b --- /dev/null +++ b/src/everos/infra/persistence/milvus/milvus_manager.py @@ -0,0 +1,105 @@ +"""Milvus connection and collection management for the derived index.""" + +from __future__ import annotations + +import re + +from pymilvus import MilvusClient + +from everos.config import MilvusSettings, load_settings +from everos.core.errors import ConfigurationError +from everos.core.observability.logging import get_logger + +logger = get_logger(__name__) + +_client: MilvusClient | None = None + + +class MilvusSchemaMismatchError(RuntimeError): + """Raised when an existing Milvus collection does not match EverOS.""" + + +class MilvusConfigurationError(ConfigurationError): + """Raised when the remote Milvus profile is incomplete or invalid.""" + + +def collection_name(table_name: str, settings: MilvusSettings | None = None) -> str: + """Return the configured Milvus collection name for an EverOS table.""" + cfg = settings or load_settings().milvus + prefix = _sanitize_name_part(cfg.collection_prefix) + base = _sanitize_name_part(table_name) + name = f"{prefix}_{base}" if prefix else base + if not re.match(r"^[A-Za-z_]", name): + name = f"_{name}" + return name + + +async def get_client() -> MilvusClient: + """Return the process-wide MilvusClient, creating it lazily.""" + global _client + if _client is None: + settings = load_settings().milvus + uri = _resolve_uri(settings) + token = _secret(settings.token) + db_name = settings.db_name or "" + _client = MilvusClient(uri=uri, token=token, db_name=db_name) + logger.info( + "milvus_connection_opened", + uri=uri, + db_name=db_name or None, + consistency_level=settings.consistency_level, + ) + return _client + + +async def dispose_connection() -> None: + """Close the process-wide Milvus client.""" + global _client + if _client is not None: + _client.close() + _client = None + logger.info("milvus_connection_closed") + from .repository import MilvusRepoBase + + MilvusRepoBase._reset_collection_cache() + + +def _resolve_uri(settings: MilvusSettings) -> str: + uri = settings.uri.strip() + if not uri: + raise MilvusConfigurationError( + "[index] backend = 'milvus' requires EVEROS_MILVUS__URI (or " + "[milvus] uri) pointing to Milvus Server or Zilliz Cloud; " + "embedded Milvus Lite is not supported" + ) + scheme, separator, _rest = uri.partition("://") + if not separator or scheme.lower() not in {"http", "https"}: + raise MilvusConfigurationError( + "[milvus] uri must be a remote http(s) endpoint for Milvus Server " + "or Zilliz Cloud, not a local database path; embedded Milvus Lite " + "is not supported" + ) + return uri + + +def _secret(value: object | None) -> str: + if value is None: + return "" + getter = getattr(value, "get_secret_value", None) + if callable(getter): + return getter() or "" + return str(value) + + +def _sanitize_name_part(value: str) -> str: + clean = re.sub(r"\W+", "_", value.strip()) + return clean.strip("_") + + +__all__ = [ + "MilvusConfigurationError", + "MilvusSchemaMismatchError", + "collection_name", + "dispose_connection", + "get_client", +] diff --git a/src/everos/infra/persistence/milvus/predicate.py b/src/everos/infra/persistence/milvus/predicate.py new file mode 100644 index 00000000..37f14b28 --- /dev/null +++ b/src/everos/infra/persistence/milvus/predicate.py @@ -0,0 +1,100 @@ +"""Render backend-neutral predicates as Milvus filter expressions.""" + +from __future__ import annotations + +import datetime as dt +import json +import re +from collections.abc import Collection +from typing import Final + +from everos.component.utils.datetime import ensure_utc, to_timestamp_ms +from everos.infra.persistence.predicate import ( + All, + AnyOf, + Comparison, + Contains, + In, + IsNull, + Predicate, + Scalar, +) + +_FIELD_NAME: Final[re.Pattern[str]] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_OPERATORS: Final[dict[str, str]] = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def render_predicate( + predicate: Predicate | None, + *, + datetime_fields: Collection[str] = (), +) -> str: + """Render a predicate using Milvus operators and physical field names.""" + if predicate is None: + return "" + if isinstance(predicate, Comparison): + return ( + f"{_field(predicate.field, datetime_fields)} " + f"{_OPERATORS[predicate.operator]} {_literal(predicate.value)}" + ) + if isinstance(predicate, In): + values = ", ".join(_literal(value) for value in predicate.values) + return f"{_field(predicate.field, datetime_fields)} in [{values}]" + if isinstance(predicate, Contains): + return ( + f"array_contains({_field(predicate.field, datetime_fields)}, " + f"{_literal(predicate.value)})" + ) + if isinstance(predicate, IsNull): + return f"{_field(predicate.field, datetime_fields)} is null" + if isinstance(predicate, All): + return _render_group(predicate.children, "and", datetime_fields) + if isinstance(predicate, AnyOf): + return _render_group(predicate.children, "or", datetime_fields) + raise TypeError(f"unsupported predicate: {type(predicate).__name__}") + + +def _render_group( + children: tuple[Predicate, ...], + operator: str, + datetime_fields: Collection[str], +) -> str: + rendered = [ + render_predicate(child, datetime_fields=datetime_fields) for child in children + ] + rendered = [item for item in rendered if item] + if not rendered: + return "" + if len(rendered) == 1: + return rendered[0] + return "(" + f" {operator} ".join(f"({item})" for item in rendered) + ")" + + +def _field(value: str, datetime_fields: Collection[str]) -> str: + if not _FIELD_NAME.fullmatch(value): + raise ValueError(f"invalid predicate field: {value!r}") + if value in datetime_fields: + return f"{value}_ms" + return value + + +def _literal(value: Scalar) -> str: + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return str(to_timestamp_ms(aware)) + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +__all__ = ["render_predicate"] diff --git a/src/everos/infra/persistence/milvus/repos.py b/src/everos/infra/persistence/milvus/repos.py new file mode 100644 index 00000000..9319a9d9 --- /dev/null +++ b/src/everos/infra/persistence/milvus/repos.py @@ -0,0 +1,160 @@ +"""Milvus repo singletons for EverOS derived index tables.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from everos.component.utils.datetime import from_timestamp +from everos.infra.persistence.index.predicate import all_of, eq, gt, is_null +from everos.infra.persistence.lancedb import ( + AgentCase, + AgentSkill, + AtomicFact, + Episode, + Foresight, + KnowledgeTopic, + UserProfile, +) + +from .repository import MilvusRepoBase + + +class _EpisodeRepo(MilvusRepoBase[Episode]): + schema = Episode + + async def count_by_owner( + self, + owner_id: str, + *, + app_id: str = "default", + project_id: str = "default", + parent_type: str | None = None, + ) -> int: + return await self._count_where( + all_of( + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + eq("parent_type", parent_type) if parent_type is not None else None, + ) + ) + + async def list_by_owner_after_ts( + self, + *, + owner_id: str, + after_ts: int, + parent_type: str, + app_id: str = "default", + project_id: str = "default", + columns: Sequence[str] | None = None, + limit: int | None = None, + ) -> list[Episode] | list[dict[str, Any]]: + predicate = all_of( + eq("owner_id", owner_id), + gt("timestamp", from_timestamp(after_ts)), + eq("parent_type", parent_type), + eq("app_id", app_id), + eq("project_id", project_id), + is_null("deprecated_by"), + ) + rows = await self.find_where(predicate, limit=limit or 20_000) + rows.sort(key=lambda row: row.timestamp) + if columns is None: + return rows + projection = list(dict.fromkeys([*columns, "timestamp"])) + return [{name: getattr(row, name) for name in projection} for row in rows] + + +class _AtomicFactRepo(MilvusRepoBase[AtomicFact]): + schema = AtomicFact + + +class _ForesightRepo(MilvusRepoBase[Foresight]): + schema = Foresight + + +class _AgentCaseRepo(MilvusRepoBase[AgentCase]): + schema = AgentCase + + +class _AgentSkillRepo(MilvusRepoBase[AgentSkill]): + schema = AgentSkill + + async def count_in_cluster(self, *, owner_id: str, cluster_id: str) -> int: + return await self._count_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)) + ) + + async def find_in_cluster( + self, *, owner_id: str, cluster_id: str, limit: int + ) -> list[AgentSkill]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), + limit=limit, + ) + + async def find_topk_relevant_in_cluster( + self, + *, + owner_id: str, + cluster_id: str, + query_vector: Sequence[float], + top_k: int, + ) -> list[AgentSkill]: + if not query_vector: + raise ValueError( + "query_vector must be non-empty; " + "call find_in_cluster for the scalar fallback" + ) + rows = await self.dense_search( + query_vector, + all_of(eq("owner_id", owner_id), eq("cluster_id", cluster_id)), + limit=top_k, + ) + out: list[AgentSkill] = [] + for row in rows: + rid = row.get("id") + if isinstance(rid, str) and (item := await self.get_by_id(rid)) is not None: + out.append(item) + return out + + +class _UserProfileRepo(MilvusRepoBase[UserProfile]): + schema = UserProfile + + +class _KnowledgeTopicRepo(MilvusRepoBase[KnowledgeTopic]): + schema = KnowledgeTopic + + +episode_repo = _EpisodeRepo() +atomic_fact_repo = _AtomicFactRepo() +foresight_repo = _ForesightRepo() +agent_case_repo = _AgentCaseRepo() +agent_skill_repo = _AgentSkillRepo() +user_profile_repo = _UserProfileRepo() +knowledge_topic_repo = _KnowledgeTopicRepo() + +ALL_REPOS = ( + episode_repo, + atomic_fact_repo, + foresight_repo, + agent_case_repo, + agent_skill_repo, + user_profile_repo, + knowledge_topic_repo, +) + +__all__ = [ + "ALL_REPOS", + "agent_case_repo", + "agent_skill_repo", + "atomic_fact_repo", + "episode_repo", + "foresight_repo", + "knowledge_topic_repo", + "user_profile_repo", +] diff --git a/src/everos/infra/persistence/milvus/repository.py b/src/everos/infra/persistence/milvus/repository.py new file mode 100644 index 00000000..44eaa11f --- /dev/null +++ b/src/everos/infra/persistence/milvus/repository.py @@ -0,0 +1,748 @@ +"""Milvus repository for EverOS rebuildable derived indexes.""" + +from __future__ import annotations + +import asyncio +import datetime as dt +from collections.abc import Sequence +from typing import Any, ClassVar + +from pydantic import BaseModel +from pymilvus import DataType, Function, FunctionType, MilvusClient + +from everos.component.utils.datetime import ensure_utc, from_timestamp, to_timestamp_ms +from everos.config import load_settings +from everos.core.observability.logging import get_logger +from everos.infra.persistence.index.predicate import ( + Predicate, + all_of, + eq, + one_of, +) +from everos.infra.persistence.index.schema import ( + IndexField, + IndexFieldKind, + IndexSchema, + schema_for, +) + +from .milvus_manager import MilvusSchemaMismatchError, collection_name, get_client +from .predicate import render_predicate + +logger = get_logger(__name__) + +_DUMMY_VECTOR_FIELD = "_everos_dummy_vector" +_DUMMY_VECTOR_DIMENSION = 2 +_SPARSE_SUFFIX = "__sparse" +_PRESENT_SUFFIX = "__present" +_UPDATE_FETCH_LIMIT = 10_000 + + +class MilvusValueLimitError(ValueError): + """A row exceeds a documented Milvus VARCHAR, array, or vector limit.""" + + +class MilvusRepoBase[T: BaseModel]: + """Generic Milvus repository backed by one neutral index schema.""" + + schema: type[T] + _write_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _collection_locks: ClassVar[dict[str, asyncio.Lock]] = {} + _ready_collections: ClassVar[set[str]] = set() + + @property + def index_schema(self) -> IndexSchema: + return schema_for(self.schema) + + @property + def table_name(self) -> str: + return self.index_schema.table_name + + @property + def collection_name(self) -> str: + return collection_name(self.table_name) + + @classmethod + def _write_lock(cls, name: str) -> asyncio.Lock: + return cls._write_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _collection_lock(cls, name: str) -> asyncio.Lock: + return cls._collection_locks.setdefault(name, asyncio.Lock()) + + @classmethod + def _reset_collection_cache(cls) -> None: + cls._ready_collections.clear() + cls._collection_locks.clear() + + @classmethod + def _reset_locks_for_tests(cls) -> None: + cls._write_locks.clear() + cls._reset_collection_cache() + + async def ensure_collection(self) -> None: + """Create or verify the collection once per process.""" + name = self.collection_name + if name in self._ready_collections: + return + async with self._collection_lock(name): + if name in self._ready_collections: + return + client = await get_client() + if await _run(client.has_collection, name): + await self.verify_collection() + else: + await self._create_collection(client) + self._ready_collections.add(name) + + async def _create_collection(self, client: MilvusClient) -> None: + schema = self._build_collection_schema() + index_params = client.prepare_index_params() + vector_fields = self.index_schema.vector_fields + if vector_fields: + for field in vector_fields: + index_params.add_index( + field_name=field.name, + index_type="AUTOINDEX", + metric_type="COSINE", + ) + else: + index_params.add_index( + field_name=_DUMMY_VECTOR_FIELD, + index_type="AUTOINDEX", + metric_type="COSINE", + ) + for field in self.index_schema.bm25_fields: + index_params.add_index( + field_name=_sparse_field(field), + index_type="AUTOINDEX", + metric_type="BM25", + ) + settings = load_settings().milvus + await _run( + client.create_collection, + collection_name=self.collection_name, + schema=schema, + index_params=index_params, + consistency_level=settings.consistency_level, + ) + logger.info( + "milvus_collection_created", + table=self.table_name, + collection=self.collection_name, + ) + + async def verify_collection(self) -> None: + client = await get_client() + description = await _run(client.describe_collection, self.collection_name) + actual = {field["name"] for field in description.get("fields", [])} + expected = set(self._stored_field_names()) + missing = expected - actual + stale = actual - expected + if missing or stale: + raise MilvusSchemaMismatchError( + f"Milvus collection {self.collection_name!r} schema drift: " + f"missing={sorted(missing)}, stale={sorted(stale)}. The index is " + "rebuildable from markdown; run `everos cascade rebuild`." + ) + + async def add(self, records: Sequence[T]) -> None: + if not records: + return + await self.ensure_collection() + payload = [self._to_milvus_record(record) for record in records] + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.insert, self.collection_name, payload) + + async def upsert(self, records: Sequence[T], *, by: str = "id") -> None: + if by != "id": + raise ValueError("MilvusRepoBase only supports upsert by id") + if not records: + return + await self.ensure_collection() + payload = [self._to_milvus_record(record) for record in records] + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, payload) + + async def update(self, updates: dict[str, Any], *, where: Predicate) -> None: + rows = await self._query_raw( + where, limit=_UPDATE_FETCH_LIMIT, include_vectors=True + ) + if not rows: + return + if len(rows) == _UPDATE_FETCH_LIMIT: + logger.warning( + "milvus_update_truncated", + table=self.table_name, + limit=_UPDATE_FETCH_LIMIT, + ) + patched: list[dict[str, Any]] = [] + for row in rows: + merged = dict(row) + for key, value in updates.items(): + self._write_field_value(merged, key, value) + self._validate_raw_record(merged) + patched.append(merged) + client = await get_client() + async with self._write_lock(self.collection_name): + await _run(client.upsert, self.collection_name, patched) + + async def optimize(self, *, cleanup_older_than: dt.timedelta | None = None) -> None: + """Milvus indexes and compaction are service-managed.""" + + async def rebuild_indexes(self) -> None: + """Milvus AUTOINDEX maintenance is service-managed.""" + + async def count(self) -> int: + return await self._count_where(None) + + async def _count_where(self, where: Predicate | None) -> int: + await self.ensure_collection() + client = await get_client() + rows = await _run( + client.query, + self.collection_name, + filter=self._expr(where), + output_fields=["count(*)"], + ) + return int(rows[0].get("count(*)", 0)) if rows else 0 + + async def get_by_id(self, id_value: str, *, id_field: str = "id") -> T | None: + if id_field != "id": + rows = await self.find_where(eq(id_field, id_value), limit=1) + return rows[0] if rows else None + await self.ensure_collection() + client = await get_client() + rows = await _run( + client.get, + self.collection_name, + ids=[id_value], + output_fields=self._output_fields(include_vectors=True), + ) + return self._model_from_milvus(rows[0]) if rows else None + + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[T]: + rows = await self._query_raw(where, limit=limit, include_vectors=True) + return [self._model_from_milvus(row) for row in rows] + + async def find_one_where(self, where: Predicate) -> T | None: + rows = await self.find_where(where, limit=1) + return rows[0] if rows else None + + async def find_where_paginated( + self, + where: Predicate, + *, + sort_by: str, + descending: bool = True, + page: int = 1, + page_size: int = 20, + max_fetch: int = 20_000, + ) -> tuple[list[T], int]: + total = await self._count_where(where) + raw = await self._query_raw(where, limit=max_fetch, include_vectors=True) + if total > len(raw): + logger.warning( + "milvus_find_where_paginated_truncated", + table=self.table_name, + total=total, + max_fetch=max_fetch, + ) + rows = [self._model_from_milvus(row) for row in raw] + rows.sort( + key=lambda row: _sort_value(getattr(row, sort_by, None)), + reverse=descending, + ) + offset = (page - 1) * page_size + return rows[offset : offset + page_size], total + + async def find_by_owner(self, owner_id: str, *, limit: int = 100) -> list[T]: + return await self.find_where(eq("owner_id", owner_id), limit=limit) + + async def find_by_md_path(self, md_path: str) -> T | None: + return await self.find_one_where(eq("md_path", md_path)) + + async def search( + self, + *, + vector: Sequence[float] | None = None, + where: Predicate | None = None, + limit: int = 10, + ) -> list[dict[str, Any]]: + if vector is None: + return await self._query_candidate_rows(where, limit=limit) + return await self.dense_search(vector, where, limit=limit) + + async def sparse_search( + self, + query_terms: Sequence[str], + where: Predicate | None, + *, + columns: Sequence[str] | None = None, + limit: int, + ) -> list[dict[str, Any]]: + if not query_terms: + return [] + await self.ensure_collection() + fields = list(columns or self.index_schema.bm25_fields) + if not fields: + return [] + unknown = set(fields) - set(self.index_schema.bm25_fields) + if unknown: + raise ValueError(f"unknown BM25 fields: {sorted(unknown)}") + client = await get_client() + query = " ".join(term for term in query_terms if term) + best: dict[str, dict[str, Any]] = {} + for field in fields: + results = await _run( + client.search, + self.collection_name, + data=[query], + anns_field=_sparse_field(field), + filter=self._expr(where), + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "BM25"}, + ) + for row in _first_result_set(results): + shaped = self._candidate_row_from_search(row) + score = _bm25_score_from_distance(row.get("distance")) + shaped["_score"] = score + rid = shaped.get("id") + if not isinstance(rid, str): + continue + prior = best.get(rid) + if prior is None or score > float(prior.get("_score", 0.0)): + best[rid] = shaped + return sorted( + best.values(), key=lambda item: float(item.get("_score", 0.0)), reverse=True + )[:limit] + + async def dense_search( + self, + vector: Sequence[float], + where: Predicate | None, + *, + limit: int, + vector_field: str = "vector", + ) -> list[dict[str, Any]]: + if not vector: + return [] + field = self.index_schema.field(vector_field) + if field.kind is not IndexFieldKind.DENSE_VECTOR: + raise ValueError(f"{vector_field!r} is not a dense-vector field") + self._validate_vector(field, vector) + await self.ensure_collection() + client = await get_client() + present = eq(_present_field(vector_field), True) + results = await _run( + client.search, + self.collection_name, + data=[list(vector)], + anns_field=vector_field, + filter=self._expr(all_of(where, present)), + limit=limit, + output_fields=self._output_fields(include_vectors=False), + search_params={"metric_type": "COSINE"}, + ) + return [ + self._candidate_row_from_search(row, normalize_cosine=True) + for row in _first_result_set(results) + ] + + async def delete(self, predicate: Predicate) -> None: + await self.ensure_collection() + client = await get_client() + async with self._write_lock(self.collection_name): + await _run( + client.delete, + self.collection_name, + filter=self._expr(predicate), + ) + + async def delete_by_md_path(self, md_path: str) -> int: + predicate = eq("md_path", md_path) + count = await self._count_where(predicate) + if count: + await self.delete(predicate) + return count + + async def find_by_owner_entry( + self, + owner_id: str, + entry_id: str, + *, + app_id: str = "default", + project_id: str = "default", + ) -> T | None: + return await self.find_one_where( + all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), + ) + ) + + async def find_by_owner_entries( + self, + owner_id: str, + entry_ids: Sequence[str], + *, + app_id: str = "default", + project_id: str = "default", + ) -> list[T]: + if not entry_ids: + return [] + return await self.find_where( + all_of( + eq("owner_id", owner_id), + one_of("entry_id", list(entry_ids)), + eq("app_id", app_id), + eq("project_id", project_id), + ), + limit=len(entry_ids), + ) + + async def find_by_session( + self, owner_id: str, session_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("owner_id", owner_id), eq("session_id", session_id)), + limit=limit, + ) + + async def find_by_parent( + self, parent_type: str, parent_id: str, *, limit: int = 100 + ) -> list[T]: + return await self.find_where( + all_of(eq("parent_type", parent_type), eq("parent_id", parent_id)), + limit=limit, + ) + + def _build_collection_schema(self): # type: ignore[no-untyped-def] + schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False) + for field in self.index_schema.fields: + self._add_schema_field(schema, field) + if not self.index_schema.vector_fields: + schema.add_field( + field_name=_DUMMY_VECTOR_FIELD, + datatype=DataType.FLOAT_VECTOR, + dim=_DUMMY_VECTOR_DIMENSION, + ) + for field in self.index_schema.bm25_fields: + sparse_name = _sparse_field(field) + schema.add_field( + field_name=sparse_name, + datatype=DataType.SPARSE_FLOAT_VECTOR, + ) + schema.add_function( + Function( + name=f"{field}_bm25", + function_type=FunctionType.BM25, + input_field_names=[field], + output_field_names=[sparse_name], + ) + ) + return schema + + def _add_schema_field(self, schema: Any, field: IndexField) -> None: + if field.kind is IndexFieldKind.STRING: + kwargs: dict[str, Any] = { + "field_name": field.name, + "datatype": DataType.VARCHAR, + "max_length": field.max_length, + } + if field.primary: + kwargs["is_primary"] = True + elif field.nullable and field.name not in self.index_schema.bm25_fields: + kwargs["nullable"] = True + if field.name in self.index_schema.bm25_fields: + kwargs["enable_analyzer"] = True + schema.add_field(**kwargs) + elif field.kind is IndexFieldKind.STRING_ARRAY: + schema.add_field( + field_name=field.name, + datatype=DataType.ARRAY, + element_type=DataType.VARCHAR, + max_capacity=field.max_capacity, + max_length=field.max_length, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.FLOAT: + schema.add_field( + field_name=field.name, + datatype=DataType.DOUBLE, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.INTEGER: + schema.add_field( + field_name=field.name, + datatype=DataType.INT64, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.DATETIME: + schema.add_field( + field_name=_datetime_storage_field(field.name), + datatype=DataType.INT64, + nullable=field.nullable, + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + schema.add_field( + field_name=field.name, + datatype=DataType.FLOAT_VECTOR, + dim=field.dimension, + ) + schema.add_field( + field_name=_present_field(field.name), + datatype=DataType.BOOL, + ) + else: # pragma: no cover - enum exhaustiveness guard + raise TypeError(f"unsupported index field kind: {field.kind}") + + def _stored_field_names(self) -> list[str]: + names: list[str] = [] + for field in self.index_schema.fields: + if field.kind is IndexFieldKind.DATETIME: + names.append(_datetime_storage_field(field.name)) + else: + names.append(field.name) + if field.kind is IndexFieldKind.DENSE_VECTOR: + names.append(_present_field(field.name)) + if not self.index_schema.vector_fields: + names.append(_DUMMY_VECTOR_FIELD) + names.extend(_sparse_field(field) for field in self.index_schema.bm25_fields) + return names + + def _output_fields(self, *, include_vectors: bool) -> list[str]: + fields: list[str] = [] + vector_names = {field.name for field in self.index_schema.vector_fields} + present_names = {_present_field(name) for name in vector_names} + for name in self._stored_field_names(): + if name.endswith(_SPARSE_SUFFIX): + continue + if name == _DUMMY_VECTOR_FIELD: + if include_vectors: + fields.append(name) + continue + if not include_vectors and (name in vector_names or name in present_names): + continue + fields.append(name) + return fields + + def _to_milvus_record(self, record: T) -> dict[str, Any]: + raw = record.model_dump(mode="python") + out: dict[str, Any] = {} + for field in self.index_schema.fields: + value = raw.get(field.name) + if field.name in self.index_schema.bm25_fields and value is None: + value = "" + if field.kind is IndexFieldKind.DATETIME: + out[_datetime_storage_field(field.name)] = ( + _datetime_to_ms(value) if value is not None else None + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + out[field.name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + out[_present_field(field.name)] = present + elif field.kind is IndexFieldKind.STRING_ARRAY: + out[field.name] = [str(item) for item in (value or [])] + else: + out[field.name] = value + if not self.index_schema.vector_fields: + out[_DUMMY_VECTOR_FIELD] = [0.0] * _DUMMY_VECTOR_DIMENSION + self._validate_raw_record(out) + return out + + def _validate_raw_record(self, row: dict[str, Any]) -> None: + for field in self.index_schema.fields: + storage_name = ( + _datetime_storage_field(field.name) + if field.kind is IndexFieldKind.DATETIME + else field.name + ) + value = row.get(storage_name) + if value is None: + if not field.nullable: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} cannot be null" + ) + continue + if field.kind is IndexFieldKind.STRING: + size = len(str(value).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} is {size} UTF-8 bytes; " + f"Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.STRING_ARRAY: + if field.max_capacity is not None and len(value) > field.max_capacity: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has {len(value)} items; " + f"Milvus limit is {field.max_capacity}" + ) + for position, item in enumerate(value): + size = len(str(item).encode("utf-8")) + if field.max_length is not None and size > field.max_length: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name}[{position}] is {size} " + f"UTF-8 bytes; Milvus limit is {field.max_length}" + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + self._validate_vector(field, value) + + def _validate_vector(self, field: IndexField, value: Sequence[float]) -> None: + if len(value) != field.dimension: + raise MilvusValueLimitError( + f"{self.table_name}.{field.name} has dimension {len(value)}; " + f"expected {field.dimension}" + ) + + def _model_from_milvus(self, row: dict[str, Any]) -> T: + return self.schema.model_validate(self._restore_row(row)) + + def _candidate_row_from_search( + self, row: dict[str, Any], *, normalize_cosine: bool = False + ) -> dict[str, Any]: + shaped = self._restore_row(row.get("entity", {})) + raw_distance = row.get("distance") + shaped["_distance"] = ( + _cosine_distance_from_milvus(raw_distance) + if normalize_cosine + else raw_distance + ) + return shaped + + def _restore_row(self, row: dict[str, Any]) -> dict[str, Any]: + out: dict[str, Any] = {} + for field in self.index_schema.fields: + if field.kind is IndexFieldKind.DATETIME: + value = row.get(_datetime_storage_field(field.name)) + out[field.name] = None if value is None else from_timestamp(int(value)) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + if field.name in row: + out[field.name] = ( + row[field.name] + if row.get(_present_field(field.name), True) + else None + ) + elif field.name in row: + value = row[field.name] + if ( + field.name in self.index_schema.bm25_fields + and value == "" + and field.nullable + ): + value = None + out[field.name] = value + return out + + def _write_field_value( + self, row: dict[str, Any], field_name: str, value: Any + ) -> None: + field = self.index_schema.field(field_name) + if field.kind is IndexFieldKind.DATETIME: + row[_datetime_storage_field(field_name)] = ( + _datetime_to_ms(value) if value is not None else None + ) + elif field.kind is IndexFieldKind.DENSE_VECTOR: + present = value is not None + row[field_name] = ( + list(value) if present else [0.0] * int(field.dimension or 0) + ) + row[_present_field(field_name)] = present + else: + row[field_name] = value + + async def _query_raw( + self, + where: Predicate | None, + *, + limit: int, + include_vectors: bool, + ) -> list[dict[str, Any]]: + await self.ensure_collection() + client = await get_client() + return await _run( + client.query, + self.collection_name, + filter=self._expr(where), + output_fields=self._output_fields(include_vectors=include_vectors), + limit=limit, + ) + + async def _query_candidate_rows( + self, where: Predicate | None, *, limit: int + ) -> list[dict[str, Any]]: + rows = await self._query_raw(where, limit=limit, include_vectors=False) + return [self._restore_row(row) for row in rows] + + def _expr(self, where: Predicate | None) -> str: + if where is not None and not isinstance(where, Predicate): + raise TypeError( + "Milvus repository predicates must use the neutral Predicate AST, " + f"got {type(where).__name__}" + ) + return render_predicate( + where, + datetime_fields=self.index_schema.datetime_fields, + ) + + +def _datetime_storage_field(name: str) -> str: + return f"{name}_ms" + + +def _present_field(name: str) -> str: + return f"{name}{_PRESENT_SUFFIX}" + + +def _sparse_field(name: str) -> str: + return f"{name}{_SPARSE_SUFFIX}" + + +def _datetime_to_ms(value: Any) -> int: + if isinstance(value, dt.datetime): + aware = ensure_utc(value) + assert aware is not None + return to_timestamp_ms(aware) + if isinstance(value, (int, float)) and not isinstance(value, bool): + return int(value) + raise TypeError(f"expected datetime or epoch ms, got {type(value).__name__}") + + +def _sort_value(value: Any) -> Any: + if value is None: + fallback = ensure_utc(dt.datetime.min) + assert fallback is not None + return fallback + return value + + +def _bm25_score_from_distance(distance: Any) -> float: + """Milvus BM25 is higher-is-better; expose a non-negative score.""" + return 0.0 if distance is None else max(0.0, float(distance)) + + +def _cosine_distance_from_milvus(distance: Any) -> float | None: + """Convert Milvus Server / Zilliz similarity to Lance-style distance.""" + if distance is None: + return None + return min(1.0, max(0.0, 1.0 - float(distance))) + + +def _first_result_set(results: Any) -> list[dict[str, Any]]: + if not results: + return [] + return list(results[0] or []) + + +async def _run(func: Any, /, *args: Any, **kwargs: Any) -> Any: + return await asyncio.to_thread(func, *args, **kwargs) + + +__all__ = ["MilvusRepoBase", "MilvusValueLimitError"] diff --git a/src/everos/infra/persistence/predicate.py b/src/everos/infra/persistence/predicate.py new file mode 100644 index 00000000..def7ca69 --- /dev/null +++ b/src/everos/infra/persistence/predicate.py @@ -0,0 +1,138 @@ +"""Backend-neutral predicate tree shared by derived-index adapters.""" + +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from typing import Literal + +type Scalar = str | int | float | bool | dt.datetime +type ComparisonOperator = Literal["eq", "ne", "gt", "gte", "lt", "lte"] + + +class Predicate: + """Marker base class for derived-index predicates.""" + + +@dataclass(frozen=True) +class Comparison(Predicate): + field: str + operator: ComparisonOperator + value: Scalar + + +@dataclass(frozen=True) +class In(Predicate): + field: str + values: tuple[Scalar, ...] + + +@dataclass(frozen=True) +class Contains(Predicate): + field: str + value: str + + +@dataclass(frozen=True) +class IsNull(Predicate): + field: str + + +@dataclass(frozen=True) +class All(Predicate): + children: tuple[Predicate, ...] + + +@dataclass(frozen=True) +class AnyOf(Predicate): + children: tuple[Predicate, ...] + + +def compare(field: str, operator: ComparisonOperator, value: Scalar) -> Predicate: + return Comparison(field, operator, value) + + +def eq(field: str, value: Scalar) -> Predicate: + return compare(field, "eq", value) + + +def ne(field: str, value: Scalar) -> Predicate: + return compare(field, "ne", value) + + +def gt(field: str, value: Scalar) -> Predicate: + return compare(field, "gt", value) + + +def gte(field: str, value: Scalar) -> Predicate: + return compare(field, "gte", value) + + +def lt(field: str, value: Scalar) -> Predicate: + return compare(field, "lt", value) + + +def lte(field: str, value: Scalar) -> Predicate: + return compare(field, "lte", value) + + +def one_of(field: str, values: list[Scalar] | tuple[Scalar, ...]) -> Predicate: + if not values: + raise ValueError("one_of requires at least one value") + return In(field, tuple(values)) + + +def contains(field: str, value: str) -> Predicate: + return Contains(field, value) + + +def is_null(field: str) -> Predicate: + return IsNull(field) + + +def all_of(*predicates: Predicate | None) -> Predicate: + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, All): + children.extend(predicate.children) + else: + children.append(predicate) + return All(tuple(children)) + + +def any_of(*predicates: Predicate | None) -> Predicate: + children: list[Predicate] = [] + for predicate in predicates: + if predicate is None: + continue + if isinstance(predicate, AnyOf): + children.extend(predicate.children) + else: + children.append(predicate) + return AnyOf(tuple(children)) + + +__all__ = [ + "All", + "AnyOf", + "Comparison", + "Contains", + "In", + "IsNull", + "Predicate", + "Scalar", + "all_of", + "any_of", + "compare", + "contains", + "eq", + "gt", + "gte", + "is_null", + "lt", + "lte", + "ne", + "one_of", +] diff --git a/src/everos/memory/cascade/_backfill.py b/src/everos/memory/cascade/_backfill.py index 767ee6d2..d78bd27b 100644 --- a/src/everos/memory/cascade/_backfill.py +++ b/src/everos/memory/cascade/_backfill.py @@ -46,8 +46,7 @@ from everos.infra.ome.config import OMEConfig from everos.infra.ome.engine import OfflineEngine from everos.infra.ome.exceptions import EngineLockHeldError -from everos.infra.persistence.lancedb import ( - BUSINESS_SCHEMAS_WITH_VECTOR, +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, @@ -58,10 +57,11 @@ agent_skill_repo, atomic_fact_repo, episode_repo, + eq, foresight_repo, - get_table, knowledge_topic_repo, ) +from everos.infra.persistence.lancedb import BUSINESS_SCHEMAS_WITH_VECTOR, get_table from everos.infra.persistence.markdown import AgentSkillFrontmatter from everos.infra.persistence.sqlite import cluster_repo, get_engine from everos.memory.cascade.worker import ( @@ -697,7 +697,7 @@ async def _backfill_table( and row.id not in subject_vectors ) try: - await backlog.spec.repo.update(updates, where=f"id = '{_q(row.id)}'") + await backlog.spec.repo.update(updates, where=eq("id", row.id)) except Exception: result.rows_failed += 1 logger.warning( diff --git a/src/everos/memory/cascade/handlers/_daily_log_base.py b/src/everos/memory/cascade/handlers/_daily_log_base.py index 8791a418..52f67b78 100644 --- a/src/everos/memory/cascade/handlers/_daily_log_base.py +++ b/src/everos/memory/cascade/handlers/_daily_log_base.py @@ -30,6 +30,7 @@ from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, StructuredEntry +from everos.infra.persistence.index import all_of, eq, one_of from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 @@ -108,7 +109,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: ] existing = await self.lance_repo.find_where( - f"md_path = '{_q(md_path)}'", + eq("md_path", md_path), limit=10_000, ) owner_id, owner_type = resolve_owner(parsed.frontmatter, md_path) @@ -198,9 +199,11 @@ async def _apply_lance_changes( if to_upsert: await self.lance_repo.upsert(to_upsert) if to_delete_ids: - in_list = ", ".join(f"'{eid}'" for eid in to_delete_ids) await self.lance_repo.delete( - f"md_path = '{_q(md_path)}' AND entry_id IN ({in_list})" + all_of( + eq("md_path", md_path), + one_of("entry_id", to_delete_ids), + ) ) async def handle_deleted(self, md_path: str) -> HandlerOutcome: @@ -252,11 +255,11 @@ async def _mark_deprecated( entry may have been deleted or not yet indexed. """ app_id, project_id = scope - predicate = ( - f"owner_id = '{_q(owner_id)}' " - f"AND entry_id = '{_q(entry_id)}' " - f"AND app_id = '{_q(app_id)}' " - f"AND project_id = '{_q(project_id)}'" + predicate = all_of( + eq("owner_id", owner_id), + eq("entry_id", entry_id), + eq("app_id", app_id), + eq("project_id", project_id), ) try: await self.lance_repo.update( @@ -290,8 +293,3 @@ async def _build_row( ``"default"`` so white-box callers exercising only the field mapping can omit them. """ - - -def _q(text: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return text.replace("'", "''") diff --git a/src/everos/memory/cascade/handlers/agent_case.py b/src/everos/memory/cascade/handlers/agent_case.py index a6fa9c5b..2f6bcca2 100644 --- a/src/everos/memory/cascade/handlers/agent_case.py +++ b/src/everos/memory/cascade/handlers/agent_case.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AgentCase, ParentType, agent_case_repo +from everos.infra.persistence.index import AgentCase, ParentType, agent_case_repo from ._common import require_float, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/agent_skill.py b/src/everos/memory/cascade/handlers/agent_skill.py index 88ae8395..1e492eec 100644 --- a/src/everos/memory/cascade/handlers/agent_skill.py +++ b/src/everos/memory/cascade/handlers/agent_skill.py @@ -42,7 +42,13 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import AgentSkill, agent_skill_repo +from everos.infra.persistence.index import ( + AgentSkill, + agent_skill_repo, + all_of, + eq, + ne, +) from everos.infra.persistence.markdown import AgentSkillFrontmatter from ..types import HandlerOutcome @@ -136,7 +142,7 @@ async def handle_added_or_modified(self, md_path: str) -> HandlerOutcome: # cascade's contract — skip the lookup. deleted = 0 if prior is None: - orphan_clause = f"md_path = '{_q(md_path)}' AND id != '{_q(skill_id)}'" + orphan_clause = all_of(eq("md_path", md_path), ne("id", skill_id)) orphans = await agent_skill_repo.find_where(orphan_clause, limit=1000) deleted = len(orphans) if deleted: @@ -224,8 +230,3 @@ def _join_body_and_references(body: str, references: str) -> str: if not body: return references return f"{body}\n\n{references}" - - -def _q(value: str) -> str: - """Defensive SQL-quote escape (mirrors lancedb chassis convention).""" - return value.replace("'", "''") diff --git a/src/everos/memory/cascade/handlers/atomic_fact.py b/src/everos/memory/cascade/handlers/atomic_fact.py index 6dea35e4..67342336 100644 --- a/src/everos/memory/cascade/handlers/atomic_fact.py +++ b/src/everos/memory/cascade/handlers/atomic_fact.py @@ -23,7 +23,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import AtomicFact, ParentType, atomic_fact_repo +from everos.infra.persistence.index import AtomicFact, ParentType, atomic_fact_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/episode.py b/src/everos/memory/cascade/handlers/episode.py index 84878d30..57dd6e5a 100644 --- a/src/everos/memory/cascade/handlers/episode.py +++ b/src/everos/memory/cascade/handlers/episode.py @@ -38,7 +38,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Episode, ParentType, episode_repo +from everos.infra.persistence.index import Episode, ParentType, episode_repo from ._common import parse_inline_list, require_iso_timestamp from ._daily_log_base import BaseDailyLogHandler, ParsedEntry diff --git a/src/everos/memory/cascade/handlers/foresight.py b/src/everos/memory/cascade/handlers/foresight.py index adbefe02..02aeea92 100644 --- a/src/everos/memory/cascade/handlers/foresight.py +++ b/src/everos/memory/cascade/handlers/foresight.py @@ -30,7 +30,7 @@ from everos.component.embedding import get_embedding_capability from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import Foresight, ParentType, foresight_repo +from everos.infra.persistence.index import Foresight, ParentType, foresight_repo from ._common import ( optional_int, diff --git a/src/everos/memory/cascade/handlers/knowledge_topic.py b/src/everos/memory/cascade/handlers/knowledge_topic.py index daa8fbc3..2535a057 100644 --- a/src/everos/memory/cascade/handlers/knowledge_topic.py +++ b/src/everos/memory/cascade/handlers/knowledge_topic.py @@ -38,7 +38,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.observability.logging import get_logger from everos.core.persistence import MarkdownReader, ParsedMarkdown -from everos.infra.persistence.lancedb import KnowledgeTopic, knowledge_topic_repo +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from everos.infra.persistence.sqlite import ( TopicUpsertPayload, knowledge_topic_sqlite_repo, diff --git a/src/everos/memory/cascade/handlers/user_profile.py b/src/everos/memory/cascade/handlers/user_profile.py index 4842986d..8d757352 100644 --- a/src/everos/memory/cascade/handlers/user_profile.py +++ b/src/everos/memory/cascade/handlers/user_profile.py @@ -27,7 +27,7 @@ from typing import Any, ClassVar from everos.core.persistence import MarkdownReader -from everos.infra.persistence.lancedb import UserProfile, user_profile_repo +from everos.infra.persistence.index import UserProfile, user_profile_repo from ..types import HandlerOutcome from ._common import content_sha256 as compute_content_sha256 diff --git a/src/everos/memory/cascade/registry.py b/src/everos/memory/cascade/registry.py index 8537575e..3054a17a 100644 --- a/src/everos/memory/cascade/registry.py +++ b/src/everos/memory/cascade/registry.py @@ -18,7 +18,7 @@ from pathlib import PurePosixPath from everos.core.persistence.markdown import BaseFrontmatter -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase, AgentSkill, AtomicFact, diff --git a/src/everos/memory/get/filters_adapter.py b/src/everos/memory/get/filters_adapter.py index 407a06bc..cd23dcc6 100644 --- a/src/everos/memory/get/filters_adapter.py +++ b/src/everos/memory/get/filters_adapter.py @@ -15,7 +15,9 @@ from __future__ import annotations +from everos.infra.persistence.index.predicate import Predicate from everos.memory.search import FilterNode, compile_filters +from everos.memory.search.filters import compile_filters_for_backends def compile_filters_for_get( @@ -25,7 +27,7 @@ def compile_filters_for_get( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: +) -> Predicate: """Compile ``/get`` filters via the shared ``compile_filters`` path. Kept as a named wrapper so ``memory.get`` consumers depend on a @@ -38,3 +40,21 @@ def compile_filters_for_get( app_id=app_id, project_id=project_id, ) + + +def compile_filters_for_get_backends( + filters: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> Predicate: + """Compile ``/get`` filters for every derived index backend.""" + return compile_filters_for_backends( + filters, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) diff --git a/src/everos/memory/get/manager.py b/src/everos/memory/get/manager.py index c42edd7e..7a942179 100644 --- a/src/everos/memory/get/manager.py +++ b/src/everos/memory/get/manager.py @@ -12,7 +12,7 @@ Reads only — never writes. Filters are compiled through :func:`compile_filters_for_get` so the column allow-list stays shared with :mod:`memory.search`. Pagination + in-memory sort -runs through :meth:`LanceRepoBase.find_where_paginated`. +runs through the configured derived index repository. """ from __future__ import annotations @@ -34,11 +34,11 @@ GetRequest, GetResponse, ) -from .filters_adapter import compile_filters_for_get +from .filters_adapter import compile_filters_for_get_backends if TYPE_CHECKING: from everos.core.persistence.lancedb import LanceRepoBase - from everos.infra.persistence.lancedb import ( + from everos.infra.persistence.index import ( AgentCase, AgentSkill, Episode, @@ -49,7 +49,7 @@ class GetManager: - """Dispatch ``GetRequest`` to the matching LanceDB-backed repo and + """Dispatch ``GetRequest`` to the matching derived index repo and shape rows into the public DTO.""" def __init__( @@ -70,7 +70,7 @@ def __init__( async def get(self, req: GetRequest) -> GetResponse: request_id = resolve_request_id() descending = req.sort_order == "desc" - where = compile_filters_for_get( + where = compile_filters_for_get_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, @@ -187,7 +187,7 @@ def _shape_agent_skill(row: AgentSkill) -> GetAgentSkillItem: async def _fetch_profile(self, owner_id: str) -> list[GetProfileItem]: """Fetch the owner's single profile row from the ``user_profile`` - LanceDB table (kept in sync with ``users//user.md`` by cascade). + Derived index row (kept in sync with ``users//user.md`` by cascade). Profile is one-row-per-owner KV — there is no pagination / sort / filter surface, so at most one item is returned. Mirrors the diff --git a/src/everos/memory/reflection/orchestrator.py b/src/everos/memory/reflection/orchestrator.py index 547adc64..9c9e8c54 100644 --- a/src/everos/memory/reflection/orchestrator.py +++ b/src/everos/memory/reflection/orchestrator.py @@ -32,6 +32,7 @@ from everos.core.observability.tracing import memory_span from everos.core.persistence import MemoryRoot from everos.infra.ome.context import StrategyContext +from everos.infra.persistence.index import all_of, eq, is_null from everos.memory._partition_locks import get_partition_lock from everos.memory.events import EpisodeExtracted @@ -41,21 +42,6 @@ _WAIT_TIMEOUT_SECONDS = 120.0 -def _escape_sql(value: str) -> str: - """Escape single quotes for LanceDB SQL-like ``where`` predicates. - - LanceDB has no parameterised query API; doubling the quote - (``'`` -> ``''``) is the SQL-standard escape. - - Args: - value: Raw string to escape. - - Returns: - Escaped string safe for interpolation into a WHERE clause. - """ - return value.replace("'", "''") - - class ReflectionOrchestrator: """Run one Reflection cycle for a single owner scope. @@ -531,12 +517,13 @@ async def _detect_orphans( app_id: Application scope. project_id: Project scope. """ - where = ( - f"parent_type = 'cluster' AND parent_id = '{_escape_sql(cluster_id)}' " - f"AND deprecated_by IS NULL " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where = all_of( + eq("parent_type", "cluster"), + eq("parent_id", cluster_id), + is_null("deprecated_by"), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ) orphans = await self._episode_store.find_where(where, limit=10) if orphans: @@ -814,14 +801,14 @@ async def _apply_deprecation_writes( to_deprecate=to_deprecate, merged_entry_id=merged_entry_id, ) - deprecated_ep_count = await self._deprecate_lance_episodes( + deprecated_ep_count = await self._deprecate_index_episodes( entry_ids=to_deprecate, owner_id=owner_id, app_id=app_id, project_id=project_id, merged_entry_id=merged_entry_id, ) - deprecated_fact_count = await self._deprecate_lance_facts( + deprecated_fact_count = await self._deprecate_index_facts( parent_ids=to_deprecate, owner_id=owner_id, merged_entry_id=merged_entry_id, @@ -848,7 +835,7 @@ async def _resolve_deprecation_targets( original_ids = {mid for mid, _ in original_members} return original_ids & current_ids - async def _deprecate_lance_episodes( + async def _deprecate_index_episodes( self, *, entry_ids: set[str], @@ -857,19 +844,19 @@ async def _deprecate_lance_episodes( project_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated episodes in LanceDB by entry_id. + """Mark deprecated episodes in the active derived index by entry_id. Returns: - Number of LanceDB update calls issued. + Number of derived-index update calls issued. """ coros: list[Any] = [ self._episode_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"entry_id = '{_escape_sql(eid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND app_id = '{_escape_sql(app_id)}' " - f"AND project_id = '{_escape_sql(project_id)}'" + where=all_of( + eq("entry_id", eid), + eq("owner_id", owner_id), + eq("app_id", app_id), + eq("project_id", project_id), ), ) for eid in entry_ids @@ -878,14 +865,14 @@ async def _deprecate_lance_episodes( await asyncio.gather(*coros) return len(coros) - async def _deprecate_lance_facts( + async def _deprecate_index_facts( self, *, parent_ids: set[str], owner_id: str, merged_entry_id: str, ) -> int: - """Mark deprecated atomic facts in LanceDB. + """Mark deprecated atomic facts in the active derived index. Args: parent_ids: Parent IDs (memcell or episode) whose facts to deprecate. @@ -893,7 +880,7 @@ async def _deprecate_lance_facts( merged_entry_id: Entry ID of the replacement merged episode. Returns: - Total number of LanceDB update calls issued. + Total number of derived-index update calls issued. """ if not parent_ids: return 0 @@ -901,10 +888,10 @@ async def _deprecate_lance_facts( coros = [ self._atomic_fact_store.update( {"deprecated_by": merged_entry_id}, - where=( - f"parent_id = '{_escape_sql(pid)}' " - f"AND owner_id = '{_escape_sql(owner_id)}' " - f"AND deprecated_by IS NULL" + where=all_of( + eq("parent_id", pid), + eq("owner_id", owner_id), + is_null("deprecated_by"), ), ) for pid in parent_ids diff --git a/src/everos/memory/search/filters.py b/src/everos/memory/search/filters.py index 2dd9aa80..74ceeb2d 100644 --- a/src/everos/memory/search/filters.py +++ b/src/everos/memory/search/filters.py @@ -1,68 +1,37 @@ -"""Filters DSL → LanceDB ``where`` string compiler. - -The Filters DSL is intentionally permissive at the JSON layer (so callers -can pass whatever they like and get a clean 400 if it is not supported) -and rigid at compile time. Field names are validated against a small -allow-list; operators against a closed enum; string literals are -single-quote-escaped. Timestamps are accepted as epoch milliseconds and -rendered as DataFusion ``TIMESTAMP ''`` literals. - -``owner_id`` and ``owner_type`` are the hard partition keys; they are -not part of the DSL at all. :func:`compile_filters` injects them at the -top of the compiled string from :class:`SearchRequest` and rejects any -attempt to override them inside ``filters``. - -Public surface --------------- - -The compiler exposes three primitives so adjacent subpackages -(notably ``memory.get``) can build narrower DSLs without forking the -field allow-list: - -* :data:`ALLOWED_FIELDS` — mapping ``field_name → _FieldSpec`` (column + - kind). Iterate / membership-test only; do not mutate. -* :data:`RESERVED_FIELDS` — names rejected inside any ``filters`` block. -* :func:`compile_predicate` — render one ``{field: value}`` clause to - SQL. Operator-map and equality-shorthand are both handled. - -The high-level :func:`compile_filters` remains the entry point for -``/search`` (combinator-aware). +"""Validate the public filter DSL and build a backend-neutral predicate. + +Storage-specific syntax is intentionally absent from this module. LanceDB and +Milvus render the resulting predicate in their own persistence packages. """ from __future__ import annotations -import datetime as _dt -from typing import Any, Final +import datetime as dt +from dataclasses import dataclass +from typing import Any, Final, Literal -from everos.component.utils.datetime import from_timestamp, to_iso_format +from everos.component.utils.datetime import ensure_utc, from_iso_format, from_timestamp from everos.core.errors import FilterError as FilterError +from everos.infra.persistence.index.predicate import ( + Predicate, + all_of, + any_of, + compare, + contains, + eq, + is_null, + one_of, +) from .dto import FilterNode -# ── Allow-lists ────────────────────────────────────────────────────────── - -_OP_MAP: Final[dict[str, str]] = { - "eq": "=", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", - "in": "IN", -} - -# Field kinds: ``str`` rendered as ``''``; ``ts`` rendered as -# ``TIMESTAMP ''`` (DataFusion timestamp literal); ``array_str`` -# uses DataFusion's ``array_has`` on a list column. -_FieldKind = str # one of: "str" | "ts" | "array_str" +_FieldKind = Literal["str", "ts", "array_str"] +@dataclass(frozen=True) class _FieldSpec: - __slots__ = ("column", "kind") - - def __init__(self, column: str, kind: _FieldKind) -> None: - self.column = column - self.kind = kind + column: str + kind: _FieldKind ALLOWED_FIELDS: Final[dict[str, _FieldSpec]] = { @@ -73,18 +42,11 @@ def __init__(self, column: str, kind: _FieldKind) -> None: "sender_id": _FieldSpec("sender_ids", "array_str"), } -# Fields the caller is explicitly **not** allowed to place inside -# ``filters``; they live at the top of :class:`SearchRequest` and are -# injected by :func:`compile_filters`. Rejecting them here turns a -# silent override into a 400. RESERVED_FIELDS: Final[frozenset[str]] = frozenset( {"owner_id", "owner_type", "app_id", "project_id"} ) -# ── Public API ─────────────────────────────────────────────────────────── - - def compile_filters( node: FilterNode | None, *, @@ -92,50 +54,53 @@ def compile_filters( owner_type: str, app_id: str = "default", project_id: str = "default", -) -> str: - """Compile a request's filters into a single LanceDB ``where`` string. - - The base clause always pins the hard partition keys (``owner_id`` / - ``owner_type`` and the ``app_id`` / ``project_id`` scope segments) to - the request's top-level values; anything in ``node`` is appended with - an ``AND``. Pinning app/project here is what isolates one space's rows - from another — omitting it would let a query bleed across spaces. Both - ``/search`` and ``/get`` share this compile path. - """ - base = [ - f"owner_id = '{_escape_str(owner_id)}'", - f"owner_type = '{owner_type}'", - f"app_id = '{_escape_str(app_id)}'", - f"project_id = '{_escape_str(project_id)}'", +) -> Predicate: + """Validate request filters and return one normalized predicate tree.""" + base: list[Predicate] = [ + eq("owner_id", owner_id), + eq("owner_type", owner_type), + eq("app_id", app_id), + eq("project_id", project_id), ] - # Only episode / atomic_fact tables carry the ``deprecated_by`` column - # (Reflection V1 marks superseded entries). Agent tables don't have it. if owner_type == "user": - base.append("deprecated_by IS NULL") - if node is None: - return " AND ".join(base) - compiled = _compile_node(node.model_dump(exclude_none=True)) - if not compiled: - return " AND ".join(base) - return " AND ".join([*base, compiled]) + base.append(is_null("deprecated_by")) + if node is not None: + compiled = _compile_node(node.model_dump(exclude_none=True)) + if compiled is not None: + base.append(compiled) + return all_of(*base) -# ── Internals ──────────────────────────────────────────────────────────── - +def compile_filters_for_backends( + node: FilterNode | None, + *, + owner_id: str, + owner_type: str, + app_id: str = "default", + project_id: str = "default", +) -> Predicate: + """Compatibility name; one predicate is rendered by the active backend.""" + return compile_filters( + node, + owner_id=owner_id, + owner_type=owner_type, + app_id=app_id, + project_id=project_id, + ) -def _compile_node(raw: dict[str, Any]) -> str: - """Walk one DSL node; return the matching SQL fragment (no leading parens). - Empty nodes yield ``""`` so :func:`compile_filters` can skip the - trailing ``AND``. - """ - raw = dict(raw) # never mutate the caller's dict - parts: list[str] = [] +def _compile_node(raw: dict[str, Any]) -> Predicate | None: + raw = dict(raw) + parts: list[Predicate] = [] if (and_list := raw.pop("AND", None)) is not None: - parts.append(_compile_combinator(and_list, "AND")) + combinator = _compile_combinator(and_list, "AND") + if combinator is not None: + parts.append(combinator) if (or_list := raw.pop("OR", None)) is not None: - parts.append(_compile_combinator(or_list, "OR")) + combinator = _compile_combinator(or_list, "OR") + if combinator is not None: + parts.append(combinator) for field, value in raw.items(): if field in RESERVED_FIELDS: @@ -146,122 +111,99 @@ def _compile_node(raw: dict[str, Any]) -> str: raise FilterError(f"unsupported filter field: {field!r}") parts.append(compile_predicate(field, value)) - # Drop empty fragments coming from empty AND/OR arrays. - parts = [p for p in parts if p] - if not parts: - return "" - if len(parts) == 1: - return parts[0] - return " AND ".join(parts) + return all_of(*parts) if parts else None -def _compile_combinator(children: list[dict[str, Any]], op: str) -> str: - """Render an ``AND`` / ``OR`` array of child nodes.""" +def _compile_combinator( + children: list[dict[str, Any]], op: Literal["AND", "OR"] +) -> Predicate | None: if not isinstance(children, list): raise FilterError(f"{op} expects an array of nodes") - fragments: list[str] = [] + fragments: list[Predicate] = [] for child in children: if not isinstance(child, dict): raise FilterError(f"{op} children must be objects") compiled = _compile_node(child) - if compiled: - fragments.append(f"({compiled})") + if compiled is not None: + fragments.append(compiled) if not fragments: - return "" - if len(fragments) == 1: - # No need for the surrounding combinator when only one effective child. - return fragments[0] - glue = f" {op} " - return "(" + glue.join(fragments) + ")" + return None + return all_of(*fragments) if op == "AND" else any_of(*fragments) -def compile_predicate(field: str, value: Any) -> str: - """Render one ``"": `` clause to SQL. - - Public primitive — :mod:`memory.get` builds a flat (no AND/OR) - DSL on top of it. Callers must pre-validate ``field`` against - :data:`ALLOWED_FIELDS` and :data:`RESERVED_FIELDS`; this function - will ``KeyError`` on unknown fields. - - ``value`` is either a scalar (equality shorthand) or an - ``{"": }`` map. Mixing multiple operators in one - dict is allowed and folds with ``AND``:: - - "timestamp": {"gte": 1, "lt": 2} - → (timestamp >= TIMESTAMP '...' AND timestamp < TIMESTAMP '...') - """ +def compile_predicate(field: str, value: Any) -> Predicate: + """Validate and normalize one field clause into the neutral AST.""" spec = ALLOWED_FIELDS[field] if isinstance(value, dict): if not value: raise FilterError(f"empty operator map for field {field!r}") - clauses = [ - _compile_op_clause(spec, field, op, op_val) for op, op_val in value.items() - ] - if len(clauses) == 1: - return clauses[0] - return "(" + " AND ".join(clauses) + ")" - # Equality shorthand. + return all_of( + *( + _compile_op_clause(spec, field, op, op_value) + for op, op_value in value.items() + ) + ) return _compile_op_clause(spec, field, "eq", value) -def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> str: - """Render a single `` `` clause.""" - if op not in _OP_MAP: +def _compile_op_clause(spec: _FieldSpec, field: str, op: str, value: Any) -> Predicate: + if op not in {"eq", "ne", "gt", "gte", "lt", "lte", "in"}: raise FilterError(f"unsupported operator {op!r} on field {field!r}") - sql_op = _OP_MAP[op] if spec.kind == "array_str": - # Only equality / membership make sense on a list column. if op == "eq": - literal = _escape_str(_require_str(value, field)) - return f"array_has({spec.column}, '{literal}')" + return contains(spec.column, _require_str(value, field)) if op == "in": - items = _require_list(value, field) - literals = [f"'{_escape_str(_require_str(v, field))}'" for v in items] - inner = " OR ".join(f"array_has({spec.column}, {lit})" for lit in literals) - return f"({inner})" + values = _require_list(value, field) + return any_of( + *(contains(spec.column, _require_str(item, field)) for item in values) + ) raise FilterError(f"operator {op!r} is not supported on array field {field!r}") if op == "in": - items = _require_list(value, field) - literals = [_render_literal(v, spec.kind, field) for v in items] - return f"{spec.column} IN ({', '.join(literals)})" - - return f"{spec.column} {sql_op} {_render_literal(value, spec.kind, field)}" - - -# ── Literal rendering ──────────────────────────────────────────────────── - - -def _render_literal(value: Any, kind: _FieldKind, field: str) -> str: + values = _require_list(value, field) + return one_of( + spec.column, + [_normalize_literal(item, spec.kind, field) for item in values], + ) + return compare( + spec.column, + op, # type: ignore[arg-type] + _normalize_literal(value, spec.kind, field), + ) + + +def _normalize_literal(value: Any, kind: _FieldKind, field: str): # type: ignore[no-untyped-def] if kind == "str": - return f"'{_escape_str(_require_str(value, field))}'" + return _require_str(value, field) if kind == "ts": - return f"TIMESTAMP '{_render_ts(value, field)}'" + return _normalize_timestamp(value, field) raise FilterError(f"unsupported field kind {kind!r} for field {field!r}") -def _render_ts(value: Any, field: str) -> str: - """Accept epoch ms (int / float) or an ISO 8601 string; emit ISO.""" - if isinstance(value, bool): # bools subclass int — reject early +def _normalize_timestamp(value: Any, field: str) -> dt.datetime: + if isinstance(value, bool): raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") - if isinstance(value, (int, float)): - return to_iso_format(from_timestamp(int(value))) - if isinstance(value, str): - # Trust the caller-supplied ISO string but escape quotes defensively. - if "'" in value: - raise FilterError(f"timestamp string for {field!r} contains a quote") - return value - if isinstance(value, _dt.datetime): - return to_iso_format(value) + try: + if isinstance(value, (int, float)): + return from_timestamp(int(value)) + if isinstance(value, str): + if "'" in value: + raise FilterError(f"timestamp string for {field!r} contains a quote") + parsed = ensure_utc(from_iso_format(value)) + assert parsed is not None + return parsed + if isinstance(value, dt.datetime): + parsed = ensure_utc(value) + assert parsed is not None + return parsed + except (TypeError, ValueError) as exc: + raise FilterError( + f"timestamp value for {field!r} must be ms or ISO string" + ) from exc raise FilterError(f"timestamp value for {field!r} must be ms or ISO string") -def _escape_str(value: str) -> str: - """Double single quotes — SQL-standard escape for a single-quoted literal.""" - return value.replace("'", "''") - - def _require_str(value: Any, field: str) -> str: if not isinstance(value, str): raise FilterError(f"value for {field!r} must be a string") @@ -272,3 +214,12 @@ def _require_list(value: Any, field: str) -> list[Any]: if not isinstance(value, list) or not value: raise FilterError(f"value for {field!r} with 'in' must be a non-empty list") return value + + +__all__ = [ + "ALLOWED_FIELDS", + "RESERVED_FIELDS", + "compile_filters", + "compile_filters_for_backends", + "compile_predicate", +] diff --git a/src/everos/memory/search/manager.py b/src/everos/memory/search/manager.py index 1ae5bfd9..7bb15a6b 100644 --- a/src/everos/memory/search/manager.py +++ b/src/everos/memory/search/manager.py @@ -22,7 +22,8 @@ no query-relevance score we can assign to a fact pulled by parent_id alone, and emitting ``score=0.0`` facts would muddy the contract. -The manager never writes to storage; it only reads LanceDB + markdown. +The manager never writes to storage; it only reads the derived index + +markdown. """ from __future__ import annotations @@ -70,7 +71,7 @@ SearchResponse, UnprocessedMessageDTO, ) -from .filters import compile_filters +from .filters import compile_filters_for_backends from .hierarchy import build_ep_to_fact_parents, heap_expand from .shaper import ( reshape_hybrid_output, @@ -204,7 +205,7 @@ async def search(self, req: SearchRequest) -> SearchResponse: # Compile filters first: a malformed `filters` payload is a user # input error (422) and should surface before the server-side # component guard (500). The two steps are independent. - where = compile_filters( + where = compile_filters_for_backends( req.filters, owner_id=req.owner_id, owner_type=req.owner_type, diff --git a/src/everos/memory/search/recall/agent_case.py b/src/everos/memory/search/recall/agent_case.py index 93a7f9b6..9c17ba45 100644 --- a/src/everos/memory/search/recall/agent_case.py +++ b/src/everos/memory/search/recall/agent_case.py @@ -13,17 +13,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentCase, get_table +from everos.infra.persistence.index import AgentCase, agent_case_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -50,43 +48,15 @@ async def sparse_recall( per BM25 column (``MatchQuery`` is column-bound), then the two per-column result lists merge by id keeping the max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentCase.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentCase.BM25_FIELDS), + merged_rows = await agent_case_repo.sparse_search( + terms, + where, + columns=AgentCase.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep the max BM25 score across the two columns. - # task_intent hits typically score higher (the retrieval anchor); - # approach hits catch queries that match a step detail. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -97,15 +67,7 @@ async def dense_recall( ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentCase.TABLE_NAME, AgentCase) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_case_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/agent_skill.py b/src/everos/memory/search/recall/agent_skill.py index f444e4a3..32408482 100644 --- a/src/everos/memory/search/recall/agent_skill.py +++ b/src/everos/memory/search/recall/agent_skill.py @@ -9,26 +9,27 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import AgentSkill, get_table +from everos.infra.persistence.index import ( + AgentSkill, + Predicate, + agent_skill_repo, + all_of, + any_of, + contains, +) from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) -def _q(value: str) -> str: - return value.replace("'", "''") - - class AgentSkillRecaller: """BM25 + vector recall over the LanceDB ``agent_skill`` table.""" @@ -40,7 +41,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Dual-column BM25 recall via OR-mode BooleanQuery per column. @@ -48,60 +49,26 @@ async def sparse_recall( rationale. One BooleanQuery per BM25 column; merge by id with max score. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, AgentSkill.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in AgentSkill.BM25_FIELDS), + merged_rows = await agent_skill_repo.sparse_search( + terms, + where, + columns=AgentSkill.BM25_FIELDS, + limit=limit, ) - # Merge by id, keep max BM25 score across the two columns. - best: dict[str, dict] = {} - for rows in per_column: - for r in rows: - rid = r.get("id") - if not isinstance(rid, str): - continue - score = float(r.get("_score", 0.0)) - existing = best.get(rid) - if existing is None or score > float(existing.get("_score", 0.0)): - merged = dict(r) - merged["_score"] = score - best[rid] = merged - merged_rows = sorted( - best.values(), key=lambda r: float(r.get("_score", 0.0)), reverse=True - )[:limit] return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await agent_skill_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -112,7 +79,7 @@ async def dense_recall( ] async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Predicate, *, limit: int ) -> list[Candidate]: """Skills whose ``source_case_ids`` intersect ``case_ids``. Filter is ``array_has`` OR-ed per id (same as @@ -124,8 +91,9 @@ async def fetch_by_case_ids( """ if not case_ids: return [] - table = await get_table(AgentSkill.TABLE_NAME, AgentSkill) - clause = " OR ".join(f"array_has(source_case_ids, '{_q(c)}')" for c in case_ids) - full_where = f"({where}) AND ({clause})" - rows = await table.query().where(full_where).limit(limit).to_list() + full_where = all_of( + where, + any_of(*(contains("source_case_ids", case_id) for case_id in case_ids)), + ) + rows = await agent_skill_repo.search(where=full_where, limit=limit) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/atomic_fact.py b/src/everos/memory/search/recall/atomic_fact.py index 69eba22d..885457e4 100644 --- a/src/everos/memory/search/recall/atomic_fact.py +++ b/src/everos/memory/search/recall/atomic_fact.py @@ -21,11 +21,16 @@ from everalgo.types import Candidate, FactCandidate -from everos.infra.persistence.lancedb import AtomicFact, get_table +from everos.infra.persistence.index import ( + AtomicFact, + Predicate, + all_of, + atomic_fact_repo, + one_of, +) from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -33,6 +38,7 @@ _NOISE_COLUMNS = frozenset( {"vector", "_distance", "_score", "created_at", "updated_at"} ) +_MAX_FACT_RECALL_LIMIT = 1024 class AtomicFactRecaller: @@ -46,17 +52,17 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery (see EpisodeRecaller docstring).""" - bq = build_or_query( - self._deps.tokenizer, query, column=AtomicFact.BM25_FIELDS[0] - ) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await atomic_fact_repo.sparse_search( + terms, + where, + columns=AtomicFact.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -64,7 +70,7 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Cosine ANN recall over the atomic_fact table. @@ -78,15 +84,7 @@ async def dense_recall( """ if not vector: return [] - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await atomic_fact_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -99,7 +97,7 @@ async def dense_recall( async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None = None, @@ -160,27 +158,22 @@ async def facts_for_episodes( async def _query_facts_for_parents( self, parent_to_eps: dict[str, list[str]], - where: str, + where: Predicate, *, per_episode: int, query_vector: Sequence[float] | None, ) -> list[dict[str, Any]]: """Construct and execute the LanceDB query for parent_id IN (...).""" - quoted = ", ".join(f"'{_q(pid)}'" for pid in parent_to_eps) - clause = f"parent_id IN ({quoted})" - full_where = f"({where}) AND ({clause})" - limit = per_episode * max(len(parent_to_eps), 1) - table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) + full_where = all_of(where, one_of("parent_id", list(parent_to_eps))) + # Milvus server / Zilliz Cloud reject search topK values above 1024. + # The fact expansion only needs a bounded candidate pool for top-N + # competition, so keep the same cap for every derived-index backend. + limit = min(per_episode * max(len(parent_to_eps), 1), _MAX_FACT_RECALL_LIMIT) if query_vector: - return await ( - table.query() - .nearest_to(list(query_vector)) - .distance_type("cosine") - .where(full_where) - .limit(limit) - .to_list() + return await atomic_fact_repo.dense_search( + query_vector, full_where, limit=limit ) - return await table.query().where(full_where).limit(limit).to_list() + return await atomic_fact_repo.search(where=full_where, limit=limit) def _build_parent_to_episode_map( @@ -193,7 +186,3 @@ def _build_parent_to_episode_map( if pid: parent_to_eps[pid].append(ep_id) return parent_to_eps - - -def _q(value: str) -> str: - return value.replace("'", "''") diff --git a/src/everos/memory/search/recall/episode.py b/src/everos/memory/search/recall/episode.py index a2766dce..f24e8ef8 100644 --- a/src/everos/memory/search/recall/episode.py +++ b/src/everos/memory/search/recall/episode.py @@ -7,11 +7,17 @@ from everalgo.types import Candidate -from everos.infra.persistence.lancedb import Episode, get_table +from everos.core.observability.logging import get_logger +from everos.infra.persistence.index import ( + Episode, + Predicate, + all_of, + episode_repo, + one_of, +) from .base import ( RecallerDeps, - build_or_query, cosine_score_from_distance, row_to_candidate, ) @@ -30,8 +36,8 @@ def _inject_parent_id(candidates: list[Candidate]) -> list[Candidate]: ] -def _q(value: str) -> str: - return value.replace("'", "''") +logger = get_logger(__name__) +_FETCH_ALL_OWNER_LIMIT = 10_000 class EpisodeRecaller: @@ -45,7 +51,7 @@ def __init__(self, deps: RecallerDeps) -> None: self._deps = deps async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """BM25 recall via OR-mode BooleanQuery. @@ -55,12 +61,14 @@ async def sparse_recall( Mirrors enterprise's ``bool.should + minimum_should_match=1`` ES design. """ - bq = build_or_query(self._deps.tokenizer, query, column=Episode.BM25_FIELDS[0]) - if bq is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query().nearest_to_text(bq).where(where).limit(limit).to_list() + rows = await episode_repo.sparse_search( + terms, + where, + columns=Episode.BM25_FIELDS, + limit=limit, ) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) @@ -68,20 +76,11 @@ async def sparse_recall( ] async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: if not vector: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await episode_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, @@ -92,19 +91,19 @@ async def dense_recall( ] async def sparse_recall_as_child( - self, query: str, where: str, *, limit: int + self, query: str, where: Predicate, *, limit: int ) -> list[Candidate]: """Sparse recall returning episodes as MaxSim child candidates.""" return _inject_parent_id(await self.sparse_recall(query, where, limit=limit)) async def dense_recall_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Dense recall (body vector ANN) returning as MaxSim children.""" return _inject_parent_id(await self.dense_recall(vector, where, limit=limit)) async def dense_recall_subject( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """ANN over the ``subject_vector`` column. @@ -113,15 +112,11 @@ async def dense_recall_subject( """ if not vector: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - rows = ( - await table.query() - .nearest_to(list(vector)) - .column("subject_vector") - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() + rows = await episode_repo.dense_search( + vector, + where, + limit=limit, + vector_field="subject_vector", ) return [ row_to_candidate( @@ -133,24 +128,28 @@ async def dense_recall_subject( ] async def dense_recall_subject_as_child( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Predicate, *, limit: int ) -> list[Candidate]: """Subject-vector ANN returning as MaxSim children.""" candidates = await self.dense_recall_subject(vector, where, limit=limit) return _inject_parent_id(candidates) - async def fetch_all_for_owner(self, where: str) -> list[Candidate]: + async def fetch_all_for_owner(self, where: Predicate) -> list[Candidate]: """Flat scan — all episodes for this owner, keyed by entry_id. Cluster membership matching in ``acluster_retrieve`` compares ``Candidate.id`` against ``Cluster.members``. Both are now episode entry_ids regardless of parent_type. - No ``limit`` — the full owner partition is required for cluster - membership matching. + The scan is capped at :data:`_FETCH_ALL_OWNER_LIMIT` to bound memory. + Hitting the cap is logged because cluster matching may be incomplete. """ - table = await get_table(Episode.TABLE_NAME, Episode) - rows = await table.query().where(where).to_list() + rows = await episode_repo.search(where=where, limit=_FETCH_ALL_OWNER_LIMIT) + if len(rows) == _FETCH_ALL_OWNER_LIMIT: + logger.warning( + "episode_owner_scan_truncated", + limit=_FETCH_ALL_OWNER_LIMIT, + ) result: list[Candidate] = [] for r in rows: entry_id = r.get("entry_id") @@ -168,13 +167,11 @@ async def fetch_all_for_owner(self, where: str) -> list[Candidate]: return result async def fetch_by_entry_ids( - self, entry_ids: list[str], where: str + self, entry_ids: list[str], where: Predicate ) -> list[Candidate]: """Fetch episodes by entry_id (for facts whose parent_id is an entry_id).""" if not entry_ids: return [] - table = await get_table(Episode.TABLE_NAME, Episode) - quoted = ", ".join(f"'{_q(eid)}'" for eid in entry_ids) - full_where = f"({where}) AND (entry_id IN ({quoted}))" - rows = await table.query().where(full_where).limit(len(entry_ids)).to_list() + full_where = all_of(where, one_of("entry_id", entry_ids)) + rows = await episode_repo.search(where=full_where, limit=len(entry_ids)) return [row_to_candidate(r, source="vector", score=0.0) for r in rows] diff --git a/src/everos/memory/search/recall/knowledge_topic.py b/src/everos/memory/search/recall/knowledge_topic.py index ea484e17..9cc6a9bf 100644 --- a/src/everos/memory/search/recall/knowledge_topic.py +++ b/src/everos/memory/search/recall/knowledge_topic.py @@ -12,17 +12,15 @@ from __future__ import annotations -import asyncio from collections.abc import Sequence from typing import ClassVar from everalgo.types import Candidate -from everos.infra.persistence.lancedb import KnowledgeTopic, get_table +from everos.infra.persistence.index import KnowledgeTopic, knowledge_topic_repo from .base import ( RecallerDeps, - build_or_query_multi_column, cosine_score_from_distance, row_to_candidate, ) @@ -78,26 +76,15 @@ async def sparse_recall( matching the query in either its summary or its content body is surfaced without double-counting. """ - column_queries = build_or_query_multi_column( - self._deps.tokenizer, query, KnowledgeTopic.BM25_FIELDS - ) - if column_queries is None: + terms = [term for term in self._deps.tokenizer.tokenize(query) if term] + if not terms: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - - async def _query_one(column: str) -> list[dict]: - return ( - await table.query() - .nearest_to_text(column_queries[column]) - .where(where) - .limit(limit) - .to_list() - ) - - per_column = await asyncio.gather( - *(_query_one(col) for col in KnowledgeTopic.BM25_FIELDS), + merged_rows = await knowledge_topic_repo.sparse_search( + terms, + where, + columns=KnowledgeTopic.BM25_FIELDS, + limit=limit, ) - merged_rows = _merge_bm25_results(per_column, limit=limit) return [ row_to_candidate(r, source="keyword", score=float(r.get("_score", 0.0))) for r in merged_rows @@ -109,15 +96,7 @@ async def dense_recall( """Cosine ANN over the ``summary`` vector (1024-d).""" if not vector: return [] - table = await get_table(KnowledgeTopic.TABLE_NAME, KnowledgeTopic) - rows = ( - await table.query() - .nearest_to(list(vector)) - .distance_type("cosine") - .where(where) - .limit(limit) - .to_list() - ) + rows = await knowledge_topic_repo.dense_search(vector, where, limit=limit) return [ row_to_candidate( r, diff --git a/src/everos/memory/search/recall/profile.py b/src/everos/memory/search/recall/profile.py index be149b92..640d2709 100644 --- a/src/everos/memory/search/recall/profile.py +++ b/src/everos/memory/search/recall/profile.py @@ -19,7 +19,7 @@ from typing import Any from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import user_profile_repo +from everos.infra.persistence.index import user_profile_repo from ..dto import SearchProfileItem diff --git a/src/everos/memory/strategies/extract_agent_skill.py b/src/everos/memory/strategies/extract_agent_skill.py index 7d4e6bc3..cb840ea3 100644 --- a/src/everos/memory/strategies/extract_agent_skill.py +++ b/src/everos/memory/strategies/extract_agent_skill.py @@ -83,10 +83,10 @@ from everos.infra.ome.context import StrategyContext from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( AgentCase as LanceAgentCase, ) -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, ) diff --git a/src/everos/memory/strategies/extract_user_profile.py b/src/everos/memory/strategies/extract_user_profile.py index 711eca09..38e7db5d 100644 --- a/src/everos/memory/strategies/extract_user_profile.py +++ b/src/everos/memory/strategies/extract_user_profile.py @@ -59,7 +59,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import BaseEvent from everos.infra.ome.triggers import Immediate -from everos.infra.persistence.lancedb import episode_repo +from everos.infra.persistence.index import episode_repo from everos.infra.persistence.markdown import ( ProfileReader, ProfileWriter, diff --git a/src/everos/memory/strategies/reflect_episodes.py b/src/everos/memory/strategies/reflect_episodes.py index 2d39b879..44aa0e14 100644 --- a/src/everos/memory/strategies/reflect_episodes.py +++ b/src/everos/memory/strategies/reflect_episodes.py @@ -22,7 +22,7 @@ from everos.infra.ome.decorator import offline_strategy from everos.infra.ome.events import CronTick from everos.infra.ome.triggers import Cron -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( atomic_fact_repo, episode_repo, ) diff --git a/src/everos/service/get.py b/src/everos/service/get.py index 6d5a72bc..0a83ff74 100644 --- a/src/everos/service/get.py +++ b/src/everos/service/get.py @@ -1,8 +1,8 @@ """Get use case — lazy singleton wiring for ``POST /api/v2/memory/get``. Mirrors :mod:`everos.service.search`: the :class:`GetManager` and its -LanceDB repo singletons are built on first call so the FastAPI module -import order stays decoupled from the lifespan that brings up LanceDB. +derived index repo singletons are built on first call so the FastAPI module +import order stays decoupled from the lifespan that brings up the index backend. ``/get`` is read-only and uses no embedding / LLM / rerank clients — it never blocks on optional components the way ``/search`` does. @@ -11,7 +11,7 @@ from __future__ import annotations from everos.core.observability.logging import get_logger -from everos.infra.persistence.lancedb import ( +from everos.infra.persistence.index import ( agent_case_repo, agent_skill_repo, episode_repo, diff --git a/src/everos/service/knowledge.py b/src/everos/service/knowledge.py index 2040ace1..6ee1a4d2 100644 --- a/src/everos/service/knowledge.py +++ b/src/everos/service/knowledge.py @@ -1148,15 +1148,15 @@ def _validate_scope_id(value: str, name: str) -> None: raise ValueError(f"{name} contains invalid characters: {value!r}") -def compile_knowledge_where(app_id: str, project_id: str) -> str: - """Build a LanceDB ``where`` clause scoped to the given tenant. +def compile_knowledge_where(app_id: str, project_id: str): # type: ignore[no-untyped-def] + """Build a backend-neutral predicate scoped to the given tenant. Args: app_id: Tenant application identifier. project_id: Tenant project identifier. Returns: - SQL-style predicate string safe for use in LanceDB ``where`` parameter. + Predicate safe for either derived-index backend. Raises: ValueError: If either id contains invalid characters. @@ -1164,10 +1164,9 @@ def compile_knowledge_where(app_id: str, project_id: str) -> str: _validate_scope_id(app_id, "app_id") _validate_scope_id(project_id, "project_id") - def _esc(v: str) -> str: - return v.replace("'", "''") + from everos.infra.persistence.index import all_of, eq - return f"app_id = '{_esc(app_id)}' AND project_id = '{_esc(project_id)}'" + return all_of(eq("app_id", app_id), eq("project_id", project_id)) # ── Recall helpers ─────────────────────────────────────────────────────────── diff --git a/tests/integration/test_cascade_cli_integration.py b/tests/integration/test_cascade_cli_integration.py index 22511ddf..c89ad7f8 100644 --- a/tests/integration/test_cascade_cli_integration.py +++ b/tests/integration/test_cascade_cli_integration.py @@ -124,7 +124,7 @@ def test_sync_with_path_outside_root_errors( # between the two tokens. output = result.stdout + (result.stderr or "") plain_output = _strip_ansi(output) - assert re.search(r"not under[^\w]+memory root", plain_output), output + assert re.search(r"not[^\w]+under[^\w]+memory[^\w]+root", plain_output), output def test_sync_with_unmatched_path( diff --git a/tests/integration/test_milvus_remote.py b/tests/integration/test_milvus_remote.py new file mode 100644 index 00000000..99734acd --- /dev/null +++ b/tests/integration/test_milvus_remote.py @@ -0,0 +1,217 @@ +"""Cross-backend behavior against Milvus Server or Zilliz Cloud. + +Set ``EVEROS_TEST_MILVUS_URI`` and, when required, +``EVEROS_TEST_MILVUS_TOKEN``. The same test is used for self-hosted and cloud +endpoints and creates uniquely prefixed, disposable collections. +""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import os +import uuid + +import pytest +import pytest_asyncio + +from everos.config import load_settings + +_URI = os.environ.get("EVEROS_TEST_MILVUS_URI", "") + +pytestmark = pytest.mark.skipif( + not _URI, + reason="EVEROS_TEST_MILVUS_URI is not configured", +) + + +@pytest_asyncio.fixture(autouse=True) +async def _remote_milvus(monkeypatch: pytest.MonkeyPatch): + prefix = f"everos_e2e_{uuid.uuid4().hex}" + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", _URI) + monkeypatch.setenv( + "EVEROS_MILVUS__TOKEN", + os.environ.get("EVEROS_TEST_MILVUS_TOKEN", ""), + ) + monkeypatch.setenv( + "EVEROS_MILVUS__DB_NAME", + os.environ.get("EVEROS_TEST_MILVUS_DB_NAME", ""), + ) + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", prefix) + load_settings.cache_clear() + + from everos.infra.persistence.index import episode_repo, startup + + try: + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + await startup() + else: + await episode_repo._repo().ensure_collection() # type: ignore[attr-defined] + yield + finally: + from everos.infra.persistence.index import drop_business_tables, shutdown + + await drop_business_tables() + await shutdown() + load_settings.cache_clear() + + +def _episode( + *, + row_id: str, + entry_id: str, + session_id: str, + text: str, + vector_axis: int, + subject_axis: int, + timestamp: dt.datetime, +): # type: ignore[no-untyped-def] + from everos.infra.persistence.index import Episode + + vector = [0.0] * 1024 + vector[vector_axis] = 1.0 + subject_vector = [0.0] * 1024 + subject_vector[subject_axis] = 1.0 + return Episode( + id=row_id, + entry_id=entry_id, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + session_id=session_id, + timestamp=timestamp, + parent_id=f"mc_{entry_id}", + sender_ids=["user"], + subject=f"subject {text}", + episode=text, + episode_tokens=text, + md_path="test_app/test_project/users/u1/episodes/day.md", + content_sha256=entry_id, + vector=vector, + subject_vector=subject_vector, + ) + + +async def test_remote_milvus_matches_derived_index_contract() -> None: + from everos.infra.persistence.index import ( + Episode, + UserProfile, + episode_repo, + eq, + user_profile_repo, + ) + from everos.memory.search import FilterNode + from everos.memory.search.filters import compile_filters + + first_vector = [1.0] + [0.0] * 1023 + first_subject = [0.0, 1.0] + [0.0] * 1022 + now = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + await episode_repo.upsert( + [ + _episode( + row_id="u1_ep1", + entry_id="ep1", + session_id="abc=", + text="red apple memory", + vector_axis=0, + subject_axis=1, + timestamp=now, + ), + _episode( + row_id="u1_ep2", + entry_id="ep2", + session_id="other", + text="blue banana memory", + vector_axis=1, + subject_axis=0, + timestamp=now + dt.timedelta(seconds=1), + ), + ] + ) + + where = compile_filters( + None, + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + rows = await episode_repo.find_where(where, limit=10) + assert {row.id for row in rows} == {"u1_ep1", "u1_ep2"} + assert await episode_repo.count() == 2 + + equals_filter = compile_filters( + FilterNode.model_validate({"session_id": "abc="}), + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + ) + equals_rows = await episode_repo.find_where(equals_filter, limit=10) + assert [row.id for row in equals_rows] == ["u1_ep1"] + + sparse = await episode_repo.sparse_search( + ["apple"], where, columns=Episode.BM25_FIELDS, limit=5 + ) + assert sparse[0]["id"] == "u1_ep1" + assert sparse[0]["_score"] > 0 + + dense = await episode_repo.dense_search(first_vector, where, limit=5) + assert dense[0]["id"] == "u1_ep1" + assert dense[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + by_subject = await episode_repo.dense_search( + first_subject, + where, + limit=5, + vector_field="subject_vector", + ) + assert by_subject[0]["id"] == "u1_ep1" + assert by_subject[0]["_distance"] == pytest.approx(0.0, abs=1e-5) + + page, total = await episode_repo.find_where_paginated( + where, + sort_by="timestamp", + page=1, + page_size=1, + max_fetch=10, + ) + assert total == 2 + assert len(page) == 1 + + concurrent = await asyncio.gather( + *(episode_repo.find_where(where, limit=10) for _ in range(4)) + ) + assert all(len(result) == 2 for result in concurrent) + + if os.environ.get("EVEROS_TEST_MILVUS_FULL_STARTUP") == "1": + profile = UserProfile( + id="u1", + owner_id="u1", + owner_type="user", + app_id="test_app", + project_id="test_project", + summary="initial profile", + explicit_info_json="[]", + implicit_traits_json="[]", + profile_timestamp_ms=1, + md_path="test_app/test_project/users/u1/user.md", + content_sha256="profile-v1", + ) + await user_profile_repo.upsert([profile]) + await user_profile_repo.update( + {"summary": "updated profile"}, where=eq("id", "u1") + ) + updated_profile = await user_profile_repo.get_by_id("u1") + assert updated_profile is not None + assert updated_profile.summary == "updated profile" + + assert ( + await episode_repo.delete_by_md_path( + "test_app/test_project/users/u1/episodes/day.md" + ) + == 2 + ) + assert await episode_repo.count() == 0 diff --git a/tests/integration/test_reflection_integration.py b/tests/integration/test_reflection_integration.py index 63b91c2f..7c7fa536 100644 --- a/tests/integration/test_reflection_integration.py +++ b/tests/integration/test_reflection_integration.py @@ -19,6 +19,7 @@ import hashlib import json from pathlib import Path +from typing import Any import numpy as np import pytest @@ -34,9 +35,11 @@ ) from everos.core.persistence.lancedb import LanceDailyLogRepoBase, LanceRepoBase from everos.infra.ome.testing import FakeStrategyContext +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.lancedb.tables.atomic_fact import AtomicFact from everos.infra.persistence.lancedb.tables.episode import Episode as LanceEpisode from everos.infra.persistence.markdown.writers.episode_writer import EpisodeWriter +from everos.infra.persistence.predicate import Predicate from everos.infra.persistence.sqlite import cluster_repo, reflection_report_repo from everos.memory._partition_locks import _reset_for_tests from everos.memory.reflection.orchestrator import ReflectionOrchestrator @@ -69,10 +72,24 @@ async def embed(self, text: str) -> list[float]: class _EpisodeRepo(LanceDailyLogRepoBase[LanceEpisode]): schema = LanceEpisode + async def find_where( + self, where: str | Predicate, *, limit: int = 100 + ) -> list[LanceEpisode]: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + return await super().find_where(rendered, limit=limit) + + async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + await super().update(updates, where=rendered) + class _AtomicFactRepo(LanceDailyLogRepoBase[AtomicFact]): schema = AtomicFact + async def update(self, updates: dict[str, Any], *, where: str | Predicate) -> None: + rendered = render_predicate(where) if isinstance(where, Predicate) else where + await super().update(updates, where=rendered) + # --------------------------------------------------------------------------- # Fixtures diff --git a/tests/integration/test_tiers/test_upgrade_path.py b/tests/integration/test_tiers/test_upgrade_path.py index 7d8c2425..72a35bb5 100644 --- a/tests/integration/test_tiers/test_upgrade_path.py +++ b/tests/integration/test_tiers/test_upgrade_path.py @@ -16,6 +16,7 @@ from __future__ import annotations +import asyncio import hashlib from pathlib import Path @@ -41,6 +42,18 @@ async def _episode_rows(owner_id: str) -> list[dict]: return await table.query().where(f"owner_id = '{owner_id}'").to_list() +async def _wait_for_episode_rows( + owner_id: str, expected: int, *, deadline_seconds: float +) -> list[dict]: + """Wait for every logical entry in a debounced daily-log update.""" + async with asyncio.timeout(deadline_seconds): + while True: + rows = await _episode_rows(owner_id) + if len(rows) >= expected: + return rows + await asyncio.sleep(0.1) + + async def _atomic_fact_rows(owner_id: str) -> list[dict]: table = await get_table(AtomicFact.TABLE_NAME, AtomicFact) return await table.query().where(f"owner_id = '{owner_id}'").to_list() @@ -118,7 +131,10 @@ async def test_tier1_to_tier2_upgrade_via_backfill( # below directly against the LanceDB table. await wait_drained(deadline_seconds=40.0) - rows = await _episode_rows("u_alice") + # The queue drains file-level work, but another append to the same + # daily log can arrive just after that drain. Wait for the logical + # entry count as the end-to-end completion condition. + rows = await _wait_for_episode_rows("u_alice", _N_ITEMS, deadline_seconds=40.0) assert len(rows) == _N_ITEMS assert all(r["vector"] is None for r in rows), ( "Tier 1 (no embed) must write every episode with vector=NULL" diff --git a/tests/unit/test_config/test_settings.py b/tests/unit/test_config/test_settings.py index fb4fbb17..6d3f2c59 100644 --- a/tests/unit/test_config/test_settings.py +++ b/tests/unit/test_config/test_settings.py @@ -120,6 +120,30 @@ def test_embedding_rerank_defaults() -> None: assert s.llm.api_key.get_secret_value() == "" +def test_index_milvus_defaults() -> None: + s = Settings() + assert s.index.backend == "lancedb" + assert s.milvus.uri == "" + assert s.milvus.token.get_secret_value() == "" + assert s.milvus.db_name == "" + assert s.milvus.consistency_level == "Session" + assert s.milvus.collection_prefix == "everos" + + +def test_index_milvus_env_overrides(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://localhost:19530") + monkeypatch.setenv("EVEROS_MILVUS__TOKEN", "secret") + monkeypatch.setenv("EVEROS_MILVUS__DB_NAME", "tenant_a") + monkeypatch.setenv("EVEROS_MILVUS__CONSISTENCY_LEVEL", "Strong") + s = Settings() + assert s.index.backend == "milvus" + assert s.milvus.uri == "http://localhost:19530" + assert s.milvus.token.get_secret_value() == "secret" + assert s.milvus.db_name == "tenant_a" + assert s.milvus.consistency_level == "Strong" + + def test_resolve_root_default(monkeypatch: pytest.MonkeyPatch) -> None: """No --root, no EVEROS_ROOT → ~/.everos.""" monkeypatch.delenv("EVEROS_ROOT", raising=False) diff --git a/tests/unit/test_infra/test_milvus/test_repo.py b/tests/unit/test_infra/test_milvus/test_repo.py new file mode 100644 index 00000000..d6fdbeb2 --- /dev/null +++ b/tests/unit/test_infra/test_milvus/test_repo.py @@ -0,0 +1,148 @@ +"""Unit coverage for the remote Milvus derived-index adapter.""" + +from __future__ import annotations + +import datetime as dt + +import pytest + +from everos.config import load_settings +from everos.infra.persistence.index import Episode, episode_repo, user_profile_repo +from everos.infra.persistence.index.schema import schema_for +from everos.infra.persistence.milvus import repository +from everos.infra.persistence.milvus.milvus_manager import ( + MilvusConfigurationError, + _resolve_uri, +) +from everos.infra.persistence.milvus.repository import ( + MilvusRepoBase, + MilvusValueLimitError, +) + + +@pytest.fixture(autouse=True) +def _reset_state(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("EVEROS_INDEX__BACKEND", "milvus") + monkeypatch.setenv("EVEROS_MILVUS__URI", "http://milvus.example:19530") + monkeypatch.setenv("EVEROS_MILVUS__COLLECTION_PREFIX", "unit_test") + load_settings.cache_clear() + MilvusRepoBase._reset_locks_for_tests() + yield + MilvusRepoBase._reset_locks_for_tests() + load_settings.cache_clear() + + +def _episode(**overrides): # type: ignore[no-untyped-def] + values = { + "id": "u1_ep1", + "entry_id": "ep1", + "owner_id": "u1", + "owner_type": "user", + "session_id": "abc=", + "timestamp": dt.datetime(2026, 1, 1, tzinfo=dt.UTC), + "parent_id": "mc1", + "sender_ids": ["user"], + "episode": "red apple memory", + "episode_tokens": "red apple memory", + "md_path": "default_app/default_project/users/u1/episodes/day.md", + "content_sha256": "a", + "vector": [1.0] + [0.0] * 1023, + "subject_vector": [0.0, 1.0] + [0.0] * 1022, + } + values.update(overrides) + return Episode(**values) + + +def test_remote_uri_is_required_and_local_paths_are_rejected() -> None: + settings = load_settings().milvus.model_copy(update={"uri": ""}) + with pytest.raises(MilvusConfigurationError, match="requires"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "/tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match="remote http"): + _resolve_uri(settings) + + settings = settings.model_copy(update={"uri": "file:///tmp/milvus.db"}) + with pytest.raises(MilvusConfigurationError, match=r"http\(s\)"): + _resolve_uri(settings) + + +def test_neutral_schema_tracks_every_model_field_and_dense_vector() -> None: + schema = schema_for(Episode) + assert {field.name for field in schema.fields} == set(Episode.model_fields) + assert [field.name for field in schema.vector_fields] == [ + "vector", + "subject_vector", + ] + + +def test_record_conversion_stores_every_dense_vector_with_presence() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + record = milvus_repo._to_milvus_record(_episode()) + assert len(record["vector"]) == 1024 + assert len(record["subject_vector"]) == 1024 + assert record["vector__present"] is True + assert record["subject_vector__present"] is True + + missing = milvus_repo._to_milvus_record(_episode(subject_vector=None)) + assert missing["subject_vector__present"] is False + assert missing["subject_vector"] == [0.0] * 1024 + + +def test_update_fetch_preserves_dummy_vector_for_scalar_only_tables() -> None: + milvus_repo = user_profile_repo._repo() # type: ignore[attr-defined] + assert "_everos_dummy_vector" in milvus_repo._output_fields(include_vectors=True) + assert "_everos_dummy_vector" not in milvus_repo._output_fields( + include_vectors=False + ) + + +def test_record_conversion_reports_varchar_array_and_vector_limits() -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + with pytest.raises(MilvusValueLimitError, match=r"episode is .* UTF-8 bytes"): + milvus_repo._to_milvus_record(_episode(episode="x" * 65_536)) + with pytest.raises(MilvusValueLimitError, match="sender_ids has 257 items"): + milvus_repo._to_milvus_record(_episode(sender_ids=["u"] * 257)) + with pytest.raises(MilvusValueLimitError, match="dimension 2"): + milvus_repo._validate_vector( + milvus_repo.index_schema.field("vector"), + [1.0, 0.0], + ) + + +def test_server_score_normalization() -> None: + assert repository._cosine_distance_from_milvus(0.75) == pytest.approx(0.25) + assert repository._bm25_score_from_distance(1.5) == pytest.approx(1.5) + + +async def test_collection_metadata_is_cached_after_startup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + milvus_repo = episode_repo._repo() # type: ignore[attr-defined] + + class _FakeClient: + has_calls = 0 + describe_calls = 0 + + def has_collection(self, name: str) -> bool: + self.has_calls += 1 + return True + + def describe_collection(self, name: str): # type: ignore[no-untyped-def] + self.describe_calls += 1 + return { + "fields": [ + {"name": field} for field in milvus_repo._stored_field_names() + ] + } + + client = _FakeClient() + + async def _fake_get_client(): # type: ignore[no-untyped-def] + return client + + monkeypatch.setattr(repository, "get_client", _fake_get_client) + await milvus_repo.ensure_collection() + await milvus_repo.ensure_collection() + assert client.has_calls == 1 + assert client.describe_calls == 1 diff --git a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py index 76990db7..8450e397 100644 --- a/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py +++ b/tests/unit/test_memory/test_cascade/test_backfill_subject_null_recovery.py @@ -29,6 +29,7 @@ from typing import Any +from everos.infra.persistence.predicate import Comparison, Predicate from everos.memory.cascade._backfill import ( NullBackfillPresenter, _backfill_table, @@ -59,9 +60,9 @@ class _RecordingRepo: assert on the exact ``{col: value}`` shape written back.""" def __init__(self) -> None: - self.updates: list[tuple[dict[str, Any], str]] = [] + self.updates: list[tuple[dict[str, Any], Predicate]] = [] - async def update(self, values: dict[str, Any], *, where: str) -> None: + async def update(self, values: dict[str, Any], *, where: Predicate) -> None: self.updates.append((values, where)) @@ -200,11 +201,12 @@ async def test_orthogonal_partial_states_all_recover() -> None: assert result.rows_processed == 3 assert result.rows_failed == 0 - # Recover per-id updates by parsing the where clause - # (``id = 'xxx'``) — the repo double preserves call order and shape. + # Recover per-id updates from the neutral predicate tree. updates_by_id: dict[str, dict[str, Any]] = {} for values, where in repo.updates: - row_id = where.split("'")[1] + assert isinstance(where, Comparison) + assert where.field == "id" + row_id = str(where.value) updates_by_id[row_id] = values assert set(updates_by_id) == {"both", "subj_only", "prim_only"} diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py index 9f79cc56..21722c0d 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_case.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_case.py @@ -24,7 +24,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import AgentCase +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import AgentCaseWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.agent_case import AgentCaseHandler @@ -61,10 +63,13 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[AgentCase] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[AgentCase]: + async def find_where( + self, where: Predicate, *, limit: int = 100 + ) -> list[AgentCase]: + rendered = render_predicate(where) prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + if rendered.startswith(prefix): + md_path = rendered[len(prefix) :].rstrip("'") return [r for r in self.rows if r.md_path == md_path] return [] @@ -75,8 +80,8 @@ async def upsert(self, rows: list[AgentCase]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: - self.deletes.append(predicate) + async def delete(self, predicate: Predicate) -> None: + self.deletes.append(render_predicate(predicate)) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) diff --git a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py index 8ed467d4..b5caa773 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py +++ b/tests/unit/test_memory/test_cascade/test_handler_agent_skill.py @@ -19,7 +19,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import AgentSkill +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import AgentSkillWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import AgentSkillHandler, HandlerDeps @@ -72,29 +74,31 @@ async def delete_by_md_path(self, md_path: str) -> int: self.deletes.append(md_path) return 1 - async def find_where(self, predicate: str, *, limit: int) -> list[AgentSkill]: + async def find_where(self, predicate: Predicate, *, limit: int) -> list[AgentSkill]: """In-memory equivalent — handles only the ``md_path = '...' AND id != '...'`` shape the handler emits.""" - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + rendered = render_predicate(predicate) + if "md_path = " in rendered and "id != " in rendered: + md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] + id_lit = rendered.split("id != '")[1].split("'", 1)[0] return [ r for r in self.rows.values() if r.md_path == md_lit and r.id != id_lit ][:limit] - raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") - - async def delete(self, predicate: str) -> None: - self.predicate_deletes.append(predicate) - if "md_path = " in predicate and "id != " in predicate: - md_lit = predicate.split("md_path = '")[1].split("'", 1)[0] - id_lit = predicate.split("id != '")[1].split("'", 1)[0] + raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") + + async def delete(self, predicate: Predicate) -> None: + rendered = render_predicate(predicate) + self.predicate_deletes.append(rendered) + if "md_path = " in rendered and "id != " in rendered: + md_lit = rendered.split("md_path = '")[1].split("'", 1)[0] + id_lit = rendered.split("id != '")[1].split("'", 1)[0] self.rows = { rid: row for rid, row in self.rows.items() if not (row.md_path == md_lit and row.id != id_lit) } return - raise NotImplementedError(f"fake repo doesn't handle {predicate!r}") + raise NotImplementedError(f"fake repo doesn't handle {rendered!r}") @pytest.fixture @@ -233,7 +237,7 @@ async def test_renaming_skill_via_frontmatter_clears_old_row( assert list(fake_repo.rows.keys()) == ["a1_new_name"] # The sweep predicate references the *new* id with the same md_path. assert fake_repo.predicate_deletes == [ - f"md_path = '{md_path}' AND id != 'a1_new_name'" + f"((md_path = '{md_path}') AND (id != 'a1_new_name'))" ] diff --git a/tests/unit/test_memory/test_cascade/test_handler_episode.py b/tests/unit/test_memory/test_cascade/test_handler_episode.py index a8af7fe1..e23855c5 100644 --- a/tests/unit/test_memory/test_cascade/test_handler_episode.py +++ b/tests/unit/test_memory/test_cascade/test_handler_episode.py @@ -19,7 +19,9 @@ from everos.component.tokenizer import Tokenizer from everos.core.persistence import MemoryRoot from everos.infra.persistence.lancedb import Episode +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.markdown import EpisodeWriter +from everos.infra.persistence.predicate import Predicate from everos.memory.cascade.handlers import HandlerDeps from everos.memory.cascade.handlers.episode import EpisodeHandler @@ -58,11 +60,12 @@ def __init__(self) -> None: self.deletes: list[str] = [] self.rows: list[Episode] = [] - async def find_where(self, where: str, *, limit: int = 100) -> list[Episode]: + async def find_where(self, where: Predicate, *, limit: int = 100) -> list[Episode]: # Honour only the md_path = '...' filter the handler emits. + rendered = render_predicate(where) prefix = "md_path = '" - if where.startswith(prefix): - md_path = where[len(prefix) :].rstrip("'") + if rendered.startswith(prefix): + md_path = rendered[len(prefix) :].rstrip("'") return [r for r in self.rows if r.md_path == md_path] return [] @@ -74,8 +77,8 @@ async def upsert(self, rows: list[Episode]) -> None: by_id[r.id] = r self.rows = list(by_id.values()) - async def delete(self, predicate: str) -> None: - self.deletes.append(predicate) + async def delete(self, predicate: Predicate) -> None: + self.deletes.append(render_predicate(predicate)) async def delete_by_md_path(self, md_path: str) -> int: before = len(self.rows) diff --git a/tests/unit/test_memory/test_get/test_filters_adapter.py b/tests/unit/test_memory/test_get/test_filters_adapter.py index 4d6c48f4..7fe6c76b 100644 --- a/tests/unit/test_memory/test_get/test_filters_adapter.py +++ b/tests/unit/test_memory/test_get/test_filters_adapter.py @@ -19,17 +19,25 @@ import pytest -from everos.memory.get.filters_adapter import compile_filters_for_get +from everos.infra.persistence.lancedb.predicate import render_predicate +from everos.memory.get.filters_adapter import ( + compile_filters_for_get as compile_filter_predicate_for_get, +) from everos.memory.search import FilterError, FilterNode +def compile_filters_for_get(*args: object, **kwargs: object) -> str: + """Render the neutral predicate with the legacy LanceDB test oracle.""" + return render_predicate(compile_filter_predicate_for_get(*args, **kwargs)) # type: ignore[arg-type] + + def test_no_filters_emits_base_clause() -> None: """``filters=None`` → owner + app/project scope clauses AND-joined.""" where = compile_filters_for_get(None, owner_id="u1", owner_type="user") assert where == ( - "owner_id = 'u1' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + "((owner_id = 'u1') AND (owner_type = 'user') " + "AND (app_id = 'default') AND (project_id = 'default') " + "AND (deprecated_by IS NULL))" ) @@ -43,9 +51,9 @@ def test_owner_id_quote_is_escaped() -> None: """SQL-standard double-quote escape on ``owner_id``.""" where = compile_filters_for_get(None, owner_id="o'reilly", owner_type="user") assert where == ( - "owner_id = 'o''reilly' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + "((owner_id = 'o''reilly') AND (owner_type = 'user') " + "AND (app_id = 'default') AND (project_id = 'default') " + "AND (deprecated_by IS NULL))" ) @@ -93,7 +101,7 @@ def test_timestamp_range_renders_and_folded() -> None: assert "timestamp < TIMESTAMP '" in where # The two clauses are AND-joined inside one parenthesised group. assert "(timestamp >= TIMESTAMP" in where - assert " AND timestamp < TIMESTAMP" in where + assert ") AND (timestamp < TIMESTAMP" in where def test_sender_id_in_list_renders_array_has() -> None: @@ -125,7 +133,7 @@ def test_top_level_and_renders_grouped_clause() -> None: ) where = compile_filters_for_get(node, owner_id="u1", owner_type="user") # Base clause is always first; combinator output appended. - assert where.startswith("owner_id = 'u1' AND owner_type = 'user' AND ") + assert where.startswith("((owner_id = 'u1') AND (owner_type = 'user') AND ") assert "session_id = 'sess_a'" in where assert "parent_id = 'mc_x'" in where diff --git a/tests/unit/test_memory/test_get/test_manager.py b/tests/unit/test_memory/test_get/test_manager.py index 35cec6b7..ea040f63 100644 --- a/tests/unit/test_memory/test_get/test_manager.py +++ b/tests/unit/test_memory/test_get/test_manager.py @@ -27,6 +27,7 @@ Episode, UserProfile, ) +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.get import ( GetManager, GetMemoryType, @@ -56,7 +57,7 @@ class _StubRepo: async def find_where_paginated( self, - where: str, + where: Any, *, sort_by: str, descending: bool = True, @@ -65,7 +66,7 @@ async def find_where_paginated( max_fetch: int = 20000, ) -> tuple[list[Any], int]: self.last = _CallRecord( - where=where, + where=render_predicate(where), sort_by=sort_by, descending=descending, page=page, diff --git a/tests/unit/test_memory/test_search/test_filters.py b/tests/unit/test_memory/test_search/test_filters.py index 808c2a38..f2c7041d 100644 --- a/tests/unit/test_memory/test_search/test_filters.py +++ b/tests/unit/test_memory/test_search/test_filters.py @@ -4,31 +4,40 @@ import pytest +from everos.infra.persistence.lancedb.predicate import render_predicate as render_lance +from everos.infra.persistence.milvus.predicate import render_predicate as render_milvus from everos.memory.search import ( FilterError, FilterNode, - compile_filters, ) +from everos.memory.search import ( + compile_filters as compile_filter_ast, +) +from everos.memory.search.filters import compile_filters_for_backends + + +def compile_filters(*args, **kwargs): # type: ignore[no-untyped-def] + """Render the neutral result through LanceDB for legacy syntax assertions.""" + return render_lance(compile_filter_ast(*args, **kwargs)) + # ── Base injection ─────────────────────────────────────────────────────── def test_no_filters_emits_base_clause() -> None: where = compile_filters(None, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" - ) + assert "owner_id = 'alice'" in where + assert "owner_type = 'user'" in where + assert "app_id = 'default'" in where + assert "project_id = 'default'" in where + assert "deprecated_by IS NULL" in where def test_no_filters_agent_omits_deprecated_by() -> None: where = compile_filters(None, owner_id="bot_42", owner_type="agent") assert "deprecated_by" not in where - assert where == ( - "owner_id = 'bot_42' AND owner_type = 'agent' " - "AND app_id = 'default' AND project_id = 'default'" - ) + assert "owner_id = 'bot_42'" in where + assert "owner_type = 'agent'" in where def test_owner_type_agent_pinned() -> None: @@ -248,10 +257,10 @@ def test_empty_and_array_skips_combinator() -> None: """Empty AND/OR arrays compile to no clauses — only the base remains.""" node = FilterNode.model_validate({"AND": []}) where = compile_filters(node, owner_id="alice", owner_type="user") - assert where == ( - "owner_id = 'alice' AND owner_type = 'user' " - "AND app_id = 'default' AND project_id = 'default' " - "AND deprecated_by IS NULL" + assert where == compile_filters( + None, + owner_id="alice", + owner_type="user", ) @@ -266,3 +275,40 @@ def test_compile_filters_excludes_deprecated_by_for_user() -> None: def test_compile_filters_omits_deprecated_by_for_agent() -> None: result = compile_filters(None, owner_id="agent_1", owner_type="agent") assert "deprecated_by" not in result + + +# ── Backend-specific rendering ───────────────────────────────────────── + + +def test_compile_filters_for_backends_preserves_lancedb_default() -> None: + filters = compile_filters_for_backends(None, owner_id="u_a", owner_type="user") + lancedb = render_lance(filters) + milvus = render_milvus(filters) + assert "owner_id = 'u_a'" in lancedb + assert 'owner_id == "u_a"' in milvus + assert "deprecated_by IS NULL" in lancedb + assert "deprecated_by is null" in milvus + + +def test_compile_filters_for_milvus_timestamp_and_array() -> None: + node = FilterNode.model_validate( + {"timestamp": {"gte": 1704067200000}, "sender_id": "u_jason"} + ) + filters = compile_filters_for_backends(node, owner_id="u_a", owner_type="user") + milvus = render_milvus(filters, datetime_fields={"timestamp"}) + lancedb = render_lance(filters) + assert "timestamp_ms >= 1704067200000" in milvus + assert 'array_contains(sender_ids, "u_jason")' in milvus + assert "TIMESTAMP '" in lancedb + assert "array_has(sender_ids, 'u_jason')" in lancedb + + +def test_compile_filters_for_milvus_escapes_string_literals() -> None: + node = FilterNode.model_validate({"session_id": "ses's"}) + filters = compile_filters_for_backends(node, owner_id="al'ice", owner_type="user") + milvus = render_milvus(filters) + lancedb = render_lance(filters) + assert 'owner_id == "al\'ice"' in milvus + assert 'session_id == "ses\'s"' in milvus + assert "owner_id = 'al''ice'" in lancedb + assert "session_id = 'ses''s'" in lancedb diff --git a/tests/unit/test_memory/test_search/test_manager.py b/tests/unit/test_memory/test_search/test_manager.py index 73dd6661..a84980d5 100644 --- a/tests/unit/test_memory/test_search/test_manager.py +++ b/tests/unit/test_memory/test_search/test_manager.py @@ -28,6 +28,7 @@ from everos.component.embedding import EmbeddingCapability from everos.component.rerank import RerankCapability from everos.core.errors import ProviderNotConfiguredError +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.memory.search.dto import SearchMethod, SearchRequest from everos.memory.search.manager import SearchManager @@ -106,25 +107,25 @@ def __init__(self, sparse: list[Candidate], dense: list[Candidate]) -> None: self.last_where: str | None = None async def sparse_recall( - self, query: str, where: str, *, limit: int + self, query: str, where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = render_predicate(where) return list(self._sparse[:limit]) async def dense_recall( - self, vector: Sequence[float], where: str, *, limit: int + self, vector: Sequence[float], where: Any, *, limit: int ) -> list[Candidate]: - self.last_where = where + self.last_where = render_predicate(where) return list(self._dense[:limit]) async def fetch_by_parent_ids( - self, parent_ids: Sequence[str], where: str + self, parent_ids: Sequence[str], where: Any ) -> list[Candidate]: by_parent = {str(c.metadata.get("parent_id", "")): c for c in self._dense} return [by_parent[p] for p in parent_ids if p in by_parent] async def fetch_by_entry_ids( - self, entry_ids: Sequence[str], where: str + self, entry_ids: Sequence[str], where: Any ) -> list[Candidate]: by_entry = {str(c.metadata.get("entry_id", "")): c for c in self._dense} return [by_entry[e] for e in entry_ids if e in by_entry] @@ -152,7 +153,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: async def facts_for_episodes( self, ep_to_parents: Mapping[str, Sequence[str]], - where: str, + where: Any, *, per_episode: int, query_vector: Any = None, @@ -203,7 +204,7 @@ async def dense_recall(self, *_: Any, **__: Any) -> list[Candidate]: return list(self._dense) async def fetch_by_case_ids( - self, case_ids: Sequence[str], where: str, *, limit: int + self, case_ids: Sequence[str], where: Any, *, limit: int ) -> list[Candidate]: return list(self._by_case) diff --git a/tests/unit/test_memory/test_search/test_recall_agent_skill.py b/tests/unit/test_memory/test_search/test_recall_agent_skill.py index 9239fa34..661e58a0 100644 --- a/tests/unit/test_memory/test_search/test_recall_agent_skill.py +++ b/tests/unit/test_memory/test_search/test_recall_agent_skill.py @@ -26,6 +26,7 @@ agent_skill_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.agent_skill import AgentSkillRecaller from everos.memory.search.recall.base import RecallerDeps @@ -76,7 +77,7 @@ def _recaller() -> AgentSkillRecaller: return AgentSkillRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) -_OWNER_WHERE = "owner_id = 'agt' AND owner_type = 'agent'" +_OWNER_WHERE = all_of(eq("owner_id", "agt"), eq("owner_type", "agent")) async def test_fetch_by_case_ids_matches_any_lineage_case() -> None: diff --git a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py index f2e80b88..6e81f755 100644 --- a/tests/unit/test_memory/test_search/test_recall_atomic_fact.py +++ b/tests/unit/test_memory/test_search/test_recall_atomic_fact.py @@ -29,6 +29,8 @@ atomic_fact_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq +from everos.memory.search.recall import atomic_fact as atomic_fact_module from everos.memory.search.recall.atomic_fact import AtomicFactRecaller from everos.memory.search.recall.base import RecallerDeps @@ -83,6 +85,9 @@ def _recaller() -> AtomicFactRecaller: return AtomicFactRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) +_ALICE_WHERE = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) + + async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: """Two episodes sharing one memcell both see the same fact pool. @@ -104,8 +109,9 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: "alice_ep_b": ["mc_shared"], "alice_ep_c": ["mc_other"], } - where = "owner_id = 'alice' AND owner_type = 'user'" - out = await _recaller().facts_for_episodes(ep_to_parents, where, per_episode=10) + out = await _recaller().facts_for_episodes( + ep_to_parents, _ALICE_WHERE, per_episode=10 + ) assert sorted(out.keys()) == ["alice_ep_a", "alice_ep_b", "alice_ep_c"] assert sorted(f.id for f in out["alice_ep_a"]) == ["alice_af_1", "alice_af_2"] @@ -120,9 +126,7 @@ async def test_facts_for_episodes_buckets_by_shared_memcell() -> None: async def test_facts_for_episodes_returns_empty_for_no_episodes() -> None: - out: dict = await _recaller().facts_for_episodes( - {}, "owner_id = 'alice'", per_episode=10 - ) + out: dict = await _recaller().facts_for_episodes({}, _ALICE_WHERE, per_episode=10) assert out == {} @@ -134,7 +138,7 @@ async def test_facts_for_episodes_skips_unknown_memcells() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"], "alice_ep_b": ["mc_missing"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert "alice_ep_a" in out @@ -163,7 +167,7 @@ async def test_facts_for_episodes_filters_by_where_clause() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert [f.id for f in out["alice_ep_a"]] == ["alice_af_1"] @@ -183,7 +187,7 @@ async def test_facts_for_episodes_drops_empty_parent_ids() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": [""]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) assert out == {} @@ -231,7 +235,7 @@ async def test_facts_for_episodes_assigns_real_cosine_score_with_query_vector() out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_shared"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, query_vector=_unit_vector(0), ) @@ -259,7 +263,7 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["mc_a"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, # no query_vector ) @@ -267,6 +271,34 @@ async def test_facts_for_episodes_score_zero_without_query_vector() -> None: assert out["alice_ep_a"][0].score == 0.0 +async def test_facts_for_episodes_caps_dense_recall_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Large parent fan-out must not exceed Milvus/Zilliz search topK limits.""" + captured: dict[str, int] = {} + + async def fake_dense_search(vector, where, *, limit): # type: ignore[no-untyped-def] + captured["limit"] = limit + return [] + + monkeypatch.setattr( + atomic_fact_module.atomic_fact_repo, + "dense_search", + fake_dense_search, + ) + ep_to_parents = {f"alice_ep_{i}": [f"parent_{i}"] for i in range(80)} + + out = await _recaller().facts_for_episodes( + ep_to_parents, + _ALICE_WHERE, + per_episode=20, + query_vector=_unit_vector(0), + ) + + assert out == {} + assert captured["limit"] == 1024 + + # ── Dual parent_id (post-1.5 migration) ──────────────────────────────── @@ -287,7 +319,7 @@ async def test_facts_for_episodes_dual_parent_id_finds_both_eras() -> None: out = await _recaller().facts_for_episodes( {"alice_ep_a": ["ep_entry_1", "mc_1"]}, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) @@ -315,7 +347,7 @@ async def test_facts_for_episodes_multiple_parent_ids_dedup_across_episodes() -> "alice_ep_a": ["ep_entry_1", "mc_shared"], "alice_ep_b": ["ep_entry_2", "mc_shared"], }, - "owner_id = 'alice' AND owner_type = 'user'", + _ALICE_WHERE, per_episode=10, ) diff --git a/tests/unit/test_memory/test_search/test_recall_episode.py b/tests/unit/test_memory/test_search/test_recall_episode.py index 4549322d..0ae51992 100644 --- a/tests/unit/test_memory/test_search/test_recall_episode.py +++ b/tests/unit/test_memory/test_search/test_recall_episode.py @@ -8,6 +8,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.predicate import eq from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.episode import EpisodeRecaller @@ -42,6 +43,9 @@ def _mock_table(rows: list[dict[str, Any]]) -> MagicMock: return tbl +_ALICE_WHERE = eq("owner_id", "alice") + + @pytest.fixture() def recaller() -> EpisodeRecaller: tok = MagicMock(spec=Tokenizer) @@ -58,11 +62,11 @@ async def test_fetch_all_for_owner_returns_entry_id_keyed_candidates( _make_row("ep_2", "mc_2"), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -75,11 +79,11 @@ async def test_fetch_all_for_owner_stores_episode_id_in_metadata( """metadata['episode_id'] carries the real LanceDB episode id for final shaping.""" rows = [_make_row("ep_abc", "mc_xyz")] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result[0].metadata["episode_id"] == "ep_abc" assert result[0].metadata["parent_id"] == "mc_xyz" @@ -104,11 +108,11 @@ async def test_fetch_all_for_owner_skips_rows_without_entry_id( }, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert result == [] @@ -131,11 +135,11 @@ async def test_fetch_all_for_owner_merged_episode_uses_entry_id( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 1 assert result[0].id == "entry_xyz", "merged episode id must be entry_id" @@ -156,11 +160,11 @@ async def test_fetch_all_for_owner_mixed_regular_and_merged( ), ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=_mock_table(rows), + return_value=rows, ): - result = await recaller.fetch_all_for_owner("owner_id = 'alice'") + result = await recaller.fetch_all_for_owner(_ALICE_WHERE) assert len(result) == 2 ids = {c.id for c in result} @@ -179,16 +183,12 @@ async def test_fetch_by_entry_ids_returns_candidates( entry_id="entry_xyz", ), ] - mock_tbl = MagicMock() - mock_tbl.query.return_value.where.return_value.limit.return_value.to_list = ( - AsyncMock(return_value=rows) - ) with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.search", new_callable=AsyncMock, - return_value=mock_tbl, + return_value=rows, ): - result = await recaller.fetch_by_entry_ids(["entry_xyz"], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids(["entry_xyz"], _ALICE_WHERE) assert len(result) == 1 assert result[0].id == "ep_merged" @@ -198,7 +198,7 @@ async def test_fetch_by_entry_ids_empty_input_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty entry_ids list short-circuits without querying.""" - result = await recaller.fetch_by_entry_ids([], "owner_id = 'alice'") + result = await recaller.fetch_by_entry_ids([], _ALICE_WHERE) assert result == [] @@ -226,13 +226,11 @@ async def test_sparse_recall_as_child_injects_parent_id( {**_make_row("ep_1", "mc_1", entry_id="entry_1"), "_score": 1.0}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table(rows), + return_value=rows, ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 assert result[0].metadata["parent_id"] == "entry_1" @@ -256,13 +254,11 @@ async def test_sparse_recall_as_child_falls_back_to_id_when_no_entry_id( "_score": 0.5, } with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.sparse_search", new_callable=AsyncMock, - return_value=_mock_bm25_table([row]), + return_value=[row], ): - result = await recaller.sparse_recall_as_child( - "hello", "owner_id = 'alice'", limit=10 - ) + result = await recaller.sparse_recall_as_child("hello", _ALICE_WHERE, limit=10) assert len(result) == 1 cand = result[0] @@ -274,7 +270,7 @@ async def test_sparse_recall_as_child_empty_query_returns_empty( ) -> None: """Empty query token list short-circuits; no table call needed.""" recaller._deps.tokenizer.tokenize.return_value = [] - result = await recaller.sparse_recall_as_child("", "owner_id = 'alice'", limit=10) + result = await recaller.sparse_recall_as_child("", _ALICE_WHERE, limit=10) assert result == [] @@ -286,12 +282,12 @@ async def test_dense_recall_as_child_injects_parent_id( {**_make_row("ep_3", "mc_3", entry_id="entry_3"), "_distance": 0.1}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_ann_table(rows), + return_value=rows, ): result = await recaller.dense_recall_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -302,7 +298,7 @@ async def test_dense_recall_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_as_child([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_as_child([], _ALICE_WHERE, limit=10) assert result == [] @@ -326,24 +322,30 @@ async def test_dense_recall_subject_returns_subject_vector_source( {**_make_row("ep_s1", "mc_s1", entry_id="entry_s1"), "_distance": 0.2}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), - ): + return_value=rows, + ) as dense_search: result = await recaller.dense_recall_subject( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 assert result[0].source == "vector" assert result[0].score == pytest.approx(0.8) + dense_search.assert_awaited_once_with( + [0.1] * 1024, + _ALICE_WHERE, + limit=10, + vector_field="subject_vector", + ) async def test_dense_recall_subject_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject([], "owner_id = 'alice'", limit=10) + result = await recaller.dense_recall_subject([], _ALICE_WHERE, limit=10) assert result == [] @@ -355,12 +357,12 @@ async def test_dense_recall_subject_as_child_injects_parent_id( {**_make_row("ep_s2", "mc_s2", entry_id="entry_s2"), "_distance": 0.15}, ] with patch( - "everos.memory.search.recall.episode.get_table", + "everos.memory.search.recall.episode.episode_repo.dense_search", new_callable=AsyncMock, - return_value=_mock_subject_ann_table(rows), + return_value=rows, ): result = await recaller.dense_recall_subject_as_child( - [0.1] * 1024, "owner_id = 'alice'", limit=10 + [0.1] * 1024, _ALICE_WHERE, limit=10 ) assert len(result) == 1 @@ -372,7 +374,5 @@ async def test_dense_recall_subject_as_child_empty_vector_returns_empty( recaller: EpisodeRecaller, ) -> None: """Empty vector short-circuits without querying.""" - result = await recaller.dense_recall_subject_as_child( - [], "owner_id = 'alice'", limit=10 - ) + result = await recaller.dense_recall_subject_as_child([], _ALICE_WHERE, limit=10) assert result == [] diff --git a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py index d40d8a41..3e98c58b 100644 --- a/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py +++ b/tests/unit/test_memory/test_search/test_recall_knowledge_topic.py @@ -1,11 +1,11 @@ """Unit tests for ``KnowledgeTopicRecaller``. -Verifies dual-column BM25 + cosine ANN recall, using ``unittest.mock`` -to patch ``get_table`` so no real LanceDB connection is needed. +Verifies dual-column BM25 delegation + cosine ANN recall, using +``unittest.mock`` to patch the backend-neutral index repo. White-box surfaces touched: - - ``everos.memory.search.recall.knowledge_topic.get_table`` (patched) - - ``KnowledgeTopicRecaller.sparse_recall`` — queries both BM25 columns + - ``everos.memory.search.recall.knowledge_topic.knowledge_topic_repo`` (patched) + - ``KnowledgeTopicRecaller.sparse_recall`` — passes both BM25 columns - ``KnowledgeTopicRecaller.dense_recall`` — cosine ANN with distance→score """ @@ -17,6 +17,7 @@ import pytest from everos.component.tokenizer import Tokenizer +from everos.infra.persistence.index import KnowledgeTopic from everos.memory.search.recall.base import RecallerDeps from everos.memory.search.recall.knowledge_topic import KnowledgeTopicRecaller @@ -58,41 +59,6 @@ def _make_row( return row -def _mock_bm25_table( - summary_rows: list[dict[str, Any]], - content_rows: list[dict[str, Any]], -) -> MagicMock: - """Build a table mock whose BM25 results differ per column. - - The first ``nearest_to_text`` call (summary_tokens) returns - ``summary_rows``; the second (content_tokens) returns ``content_rows``. - ``asyncio.gather`` fires both concurrently, so we use ``side_effect`` - on the chain rather than recording call order. - """ - summary_chain = MagicMock() - summary_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=summary_rows - ) - - content_chain = MagicMock() - content_chain.where.return_value.limit.return_value.to_list = AsyncMock( - return_value=content_rows - ) - - tbl = MagicMock() - tbl.query.return_value.nearest_to_text.side_effect = [summary_chain, content_chain] - return tbl - - -def _mock_ann_table(rows: list[dict[str, Any]]) -> MagicMock: - """Build a table mock for ANN (dense) queries.""" - tbl = MagicMock() - ann = tbl.query.return_value.nearest_to.return_value - chain = ann.distance_type.return_value.where.return_value.limit.return_value - chain.to_list = AsyncMock(return_value=rows) - return tbl - - @pytest.fixture() def recaller() -> KnowledgeTopicRecaller: return KnowledgeTopicRecaller(RecallerDeps(tokenizer=_WhitespaceTokenizer())) @@ -109,16 +75,20 @@ def recaller() -> KnowledgeTopicRecaller: async def test_sparse_recall_queries_both_columns( recaller: KnowledgeTopicRecaller, ) -> None: - """``nearest_to_text`` must be called once per BM25 column.""" - tbl = _mock_bm25_table( - summary_rows=[_make_row("t1", score=0.9)], - content_rows=[_make_row("t2", score=0.7)], - ) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + """Both BM25 columns must be delegated to the index repo.""" + rows = [_make_row("t1", score=0.9), _make_row("t2", score=0.7)] + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ) as mock_sparse: result = await recaller.sparse_recall("topic query", _WHERE, limit=10) - # nearest_to_text called twice (once per column) - assert tbl.query.return_value.nearest_to_text.call_count == 2 + mock_sparse.assert_awaited_once() + assert list(mock_sparse.await_args.args[0]) == ["topic", "query"] + assert mock_sparse.await_args.args[1] == _WHERE + assert mock_sparse.await_args.kwargs["columns"] == KnowledgeTopic.BM25_FIELDS + assert mock_sparse.await_args.kwargs["limit"] == 10 ids = {c.id for c in result} assert ids == {"t1", "t2"} @@ -126,13 +96,14 @@ async def test_sparse_recall_queries_both_columns( async def test_sparse_recall_merges_by_max_score( recaller: KnowledgeTopicRecaller, ) -> None: - """When the same id appears in both columns, keep the higher score.""" + """Scores returned by the repo are preserved on keyword candidates.""" shared_id = "topic_shared" - summary_rows = [_make_row(shared_id, score=0.5)] - content_rows = [_make_row(shared_id, score=0.9)] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=[_make_row(shared_id, score=0.9)], + ): result = await recaller.sparse_recall("overlap", _WHERE, limit=10) assert len(result) == 1 @@ -144,16 +115,16 @@ async def test_sparse_recall_merges_by_max_score( async def test_sparse_recall_returns_sorted_by_score( recaller: KnowledgeTopicRecaller, ) -> None: - """Merged results must be sorted descending by score, truncated to limit.""" - summary_rows = [ - _make_row("a", score=0.3), + """The recaller preserves repo ordering and maps rows to candidates.""" + rows = [ _make_row("b", score=0.8), - ] - content_rows = [ _make_row("c", score=0.6), ] - tbl = _mock_bm25_table(summary_rows, content_rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.sparse_recall("query", _WHERE, limit=2) assert len(result) == 2 @@ -164,16 +135,19 @@ async def test_sparse_recall_returns_sorted_by_score( async def test_sparse_recall_empty_query_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty tokenisation short-circuits — no LanceDB query is issued.""" + """Empty tokenisation short-circuits — no repo query is issued.""" tok = MagicMock(spec=Tokenizer) tok.tokenize.return_value = [] r = KnowledgeTopicRecaller(RecallerDeps(tokenizer=tok)) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + with patch( + f"{_MODULE}.knowledge_topic_repo.sparse_search", + new_callable=AsyncMock, + ) as mock_sparse: result = await r.sparse_recall("", _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_sparse.assert_not_called() # --------------------------------------------------------------------------- @@ -189,8 +163,11 @@ async def test_dense_recall_cosine_conversion( _make_row("t1", distance=0.2), _make_row("t2", distance=0.5), ] - tbl = _mock_ann_table(rows) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=rows, + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=10) assert len(result) == 2 @@ -203,12 +180,15 @@ async def test_dense_recall_cosine_conversion( async def test_dense_recall_empty_vector_returns_empty( recaller: KnowledgeTopicRecaller, ) -> None: - """Empty vector short-circuits — no LanceDB query is issued.""" - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock) as mock_gt: + """Empty vector short-circuits — no repo query is issued.""" + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + ) as mock_dense: result = await recaller.dense_recall([], _WHERE, limit=10) assert result == [] - mock_gt.assert_not_called() + mock_dense.assert_not_called() async def test_dense_recall_metadata_excludes_noise_columns( @@ -218,8 +198,11 @@ async def test_dense_recall_metadata_excludes_noise_columns( row = _make_row("t1", distance=0.3) row["vector"] = [0.0] * 1024 - tbl = _mock_ann_table([row]) - with patch(f"{_MODULE}.get_table", new_callable=AsyncMock, return_value=tbl): + with patch( + f"{_MODULE}.knowledge_topic_repo.dense_search", + new_callable=AsyncMock, + return_value=[row], + ): result = await recaller.dense_recall([0.1] * 1024, _WHERE, limit=5) assert len(result) == 1 diff --git a/tests/unit/test_memory/test_search/test_recall_or_semantics.py b/tests/unit/test_memory/test_search/test_recall_or_semantics.py index 3e35cde8..0c2fe4a7 100644 --- a/tests/unit/test_memory/test_search/test_recall_or_semantics.py +++ b/tests/unit/test_memory/test_search/test_recall_or_semantics.py @@ -30,6 +30,7 @@ episode_repo, lancedb_manager, ) +from everos.infra.persistence.predicate import all_of, eq from everos.memory.search.recall.base import RecallerDeps, build_or_query from everos.memory.search.recall.episode import EpisodeRecaller @@ -148,7 +149,7 @@ async def test_or_semantics_poison_token_does_not_kill_query() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("alice support group", where, limit=10) assert cands, "alice + support + group should recall ep_1 via SHOULD" # ep_1 is the support-group episode; should rank above ep_2 (no support). @@ -177,7 +178,7 @@ async def test_or_semantics_single_informative_token() -> None: tbl = await get_table(Episode.TABLE_NAME, Episode) await tbl.optimize() - where = "owner_id = 'alice' AND owner_type = 'user'" + where = all_of(eq("owner_id", "alice"), eq("owner_type", "user")) cands = await _recaller().sparse_recall("painting", where, limit=10) assert cands, "single informative token must recall the matching episode" assert cands[0].id == "alice_ep_2" @@ -185,5 +186,5 @@ async def test_or_semantics_single_informative_token() -> None: async def test_or_semantics_empty_query_returns_empty() -> None: """Tokenisation yields nothing → recall returns ``[]`` without hitting LanceDB.""" - cands = await _recaller().sparse_recall(" ", "owner_id = 'alice'", limit=10) + cands = await _recaller().sparse_recall(" ", eq("owner_id", "alice"), limit=10) assert cands == [] diff --git a/tests/unit/test_service/test_knowledge_search.py b/tests/unit/test_service/test_knowledge_search.py index 73500924..037b39bd 100644 --- a/tests/unit/test_service/test_knowledge_search.py +++ b/tests/unit/test_service/test_knowledge_search.py @@ -17,6 +17,7 @@ from everos.component.utils.datetime import get_utc_now from everos.core.errors import ProviderNotConfiguredError +from everos.infra.persistence.lancedb.predicate import render_predicate from everos.infra.persistence.sqlite.tables.knowledge import ( KnowledgeDocumentRow, ) @@ -134,11 +135,11 @@ def _patch_stack( class TestCompileKnowledgeWhere: def test_basic_clause(self) -> None: - result = compile_knowledge_where("myapp", "myproj") - assert result == "app_id = 'myapp' AND project_id = 'myproj'" + result = render_predicate(compile_knowledge_where("myapp", "myproj")) + assert result == "((app_id = 'myapp') AND (project_id = 'myproj'))" def test_defaults(self) -> None: - result = compile_knowledge_where("default", "default") + result = render_predicate(compile_knowledge_where("default", "default")) assert "app_id = 'default'" in result assert "project_id = 'default'" in result @@ -159,12 +160,12 @@ def test_rejects_empty_project_id(self) -> None: compile_knowledge_where("app", "") def test_accepts_valid_ids_with_special_chars(self) -> None: - result = compile_knowledge_where("my_app.v2", "project-1") + result = render_predicate(compile_knowledge_where("my_app.v2", "project-1")) assert "my_app.v2" in result assert "project-1" in result def test_accepts_valid_ids_with_at_plus(self) -> None: - result = compile_knowledge_where("app@org+v1", "proj_1") + result = render_predicate(compile_knowledge_where("app@org+v1", "proj_1")) assert "app@org+v1" in result assert "proj_1" in result diff --git a/uv.lock b/uv.lock index ec09b1ab..f0757aa7 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,14 @@ version = 1 revision = 3 requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] [[package]] name = "aiosqlite" @@ -99,6 +107,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/d2/47e8bc06fe2a06d3f5bdf20f1126ab66c4e99dc48d940e7ba873f7ac7131/cachetools-7.1.7.tar.gz", hash = "sha256:a3e2a00b14d8f8a6b70c1dae7b4685e7ad3bc965c5b42124a2d6ce895da6cf50", size = 40680, upload-time = "2026-08-01T21:20:40.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/d8/767faeda872075724b95dd675466a645f1b92aadcdcf2d1429dcfd76c176/cachetools-7.1.7-py3-none-any.whl", hash = "sha256:ef98ef375ad188819ef2f9b3645e3987f4b8c5b7550e436ad998c2de78296df0", size = 16830, upload-time = "2026-08-01T21:20:38.977Z" }, +] + [[package]] name = "cairocffi" version = "1.7.1" @@ -595,6 +612,9 @@ dependencies = [ ] [package.optional-dependencies] +milvus = [ + { name = "pymilvus" }, +] multimodal = [ { name = "everalgo-parser", extra = ["svg"] }, ] @@ -611,6 +631,7 @@ dev = [ { name = "opentelemetry-sdk" }, { name = "pre-commit" }, { name = "pyinstrument" }, + { name = "pymilvus" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -641,6 +662,7 @@ requires-dist = [ { name = "prometheus-client", specifier = ">=0.20.0" }, { name = "pydantic", specifier = ">=2.7.1" }, { name = "pydantic-settings", specifier = ">=2.0.0" }, + { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=3.0.0" }, { name = "python-multipart", specifier = ">=0.0.7" }, { name = "pyyaml", specifier = ">=6.0" }, { name = "sqlmodel", specifier = ">=0.0.22" }, @@ -651,7 +673,7 @@ requires-dist = [ { name = "watchdog", specifier = ">=4.0.0" }, { name = "watchfiles", specifier = ">=0.21.0" }, ] -provides-extras = ["multimodal", "otel"] +provides-extras = ["multimodal", "otel", "milvus"] [package.metadata.requires-dev] dev = [ @@ -661,6 +683,7 @@ dev = [ { name = "opentelemetry-sdk", specifier = ">=1.27.0" }, { name = "pre-commit", specifier = ">=4.0.0" }, { name = "pyinstrument", specifier = ">=5.0.0" }, + { name = "pymilvus", specifier = ">=3.0.0" }, { name = "pytest", specifier = ">=8.4.0" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.0.0" }, @@ -850,6 +873,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/80/8f/774ce522de6a7e70fbeceeaeb6fbe502f5dfb8365728fb3bb4cb23463da8/grimp-3.14-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a424ad14d5deb56721ac24ab939747f72ab3d378d42e7d1f038317d33b052b77", size = 2515157, upload-time = "2025-12-10T17:54:55.874Z" }, ] +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/39/33/b5b50fc2c6fbe350e04814047bb2d409feec7b36ef8b170254c050e06bc0/grpcio-1.83.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:33898e6a28e4ae598f1577cb1c4fec2a15c033d0ec52b9b45a09610dd045b9da", size = 12160538, upload-time = "2026-07-23T15:19:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/77af31228f55f55a2a5112bb0077ad0a1c4d23dbb0c2853a62475bbdcc14/grpcio-1.83.0-cp312-cp312-win32.whl", hash = "sha256:cb2906c61db4f9c64cc360054b5df70eeb81846228e9e56a4944bd415a63dadc", size = 4394004, upload-time = "2026-07-23T15:19:50.618Z" }, + { url = "https://files.pythonhosted.org/packages/c0/da/f706e39550e7a3732ce2b9c5926107a93d74a802775b19b642a6df27dc96/grpcio-1.83.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c699bbb20f143c8f2bff219de578aa2dc1f919399d67dc702b038b986ee62df", size = 5158525, upload-time = "2026-07-23T15:19:52.246Z" }, + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a1/121806ce69f23138dabe06aa595b0e5f1ae051a37e4c1954eed7d692c800/grpcio-1.83.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:74fe6f9e8a35c7dbf32255ee154d15e3e5338a81ed39173d079d594d2e544cd1", size = 12154419, upload-time = "2026-07-23T15:19:56.3Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/93/b6/d70b69ae5c0cfc341b9ba474980e4ed99cbf05c0e4a14e9eee8cb73db0a5/grpcio-1.83.0-cp313-cp313-win32.whl", hash = "sha256:8fe04f1050a59f875601eb55d42b4f66946fe89817f967e34db1462ccd07dadf", size = 4393993, upload-time = "2026-07-23T15:20:11.017Z" }, + { url = "https://files.pythonhosted.org/packages/0f/13/45d4cccb555cf4c476226979bf3d2fd0b0254216f7564c3a053e35117efc/grpcio-1.83.0-cp313-cp313-win_amd64.whl", hash = "sha256:6e01ecd9d8ef280abe1365138a4dc318f9a5287f4cb1b41d07816f796653f735", size = 5159650, upload-time = "2026-07-23T15:20:12.979Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ab/d3874931d123a95e83a3ebf8aa04537988fb62425cedb8bf3cefc5ad41b2/grpcio-1.83.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d05ff664100d429335b93c91b8b34ddf9e94a112205e7fa06dede309e44a4e4c", size = 12166617, upload-time = "2026-07-23T15:20:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, + { url = "https://files.pythonhosted.org/packages/83/fa/c3fda157287f64bc65acee6c5aa90c41acf9e0d3a8e69a265eecff6d00a1/grpcio-1.83.0-cp314-cp314-win32.whl", hash = "sha256:32e11c37f5285b0c6fa3042c05fe06903696689749833fc64e67dec71b9bbe33", size = 4471765, upload-time = "2026-07-23T15:20:33.195Z" }, + { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1446,6 +1510,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "orjson" +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/4a/295da39c651c2faac8bd351a2a346f0fdedd9d50b847ee9dfc27d2207ef6/orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0", size = 223427, upload-time = "2026-08-14T16:12:28.525Z" }, + { url = "https://files.pythonhosted.org/packages/29/98/758cf90fbeaaafb7f8141bfac75a432099959f3a2f5db93a412e876415d8/orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54", size = 123725, upload-time = "2026-08-14T16:12:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/32/b5/5b934d251f8651f7e41df180ad0c57a6e1cabe15c7bd331638413a50ebc9/orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83", size = 113375, upload-time = "2026-08-14T16:12:31.209Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d2/37efb5b12a176ce3ced29f4144f20da57d02757f78ce549637dc1b4e1fc8/orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7", size = 129983, upload-time = "2026-08-14T16:12:32.721Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/0644b87c73f13e0092df8f35a1fe280d991e5e90072087411e0dd7e44e0c/orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e", size = 130629, upload-time = "2026-08-14T16:12:34.084Z" }, + { url = "https://files.pythonhosted.org/packages/8c/57/80b986ebfecd9c6a177ddf1c2319717f0cd8feffb2b78946595a18a2fc88/orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b", size = 131245, upload-time = "2026-08-14T16:12:35.713Z" }, + { url = "https://files.pythonhosted.org/packages/80/3d/75c5ac5a69161f44492a68fbdde66f4cc4ce48cd5e1fb05918e46f0c8848/orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f", size = 135397, upload-time = "2026-08-14T16:12:37.128Z" }, + { url = "https://files.pythonhosted.org/packages/71/93/4d71f2df314a97ff0d27a4559bf5888fc8406e3c6dec90e92291e3511215/orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873", size = 127693, upload-time = "2026-08-14T16:12:38.627Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1d/0dbc6be5adfd1730491072fb60beb6bcdf5d7b2596ee41b7fc2e298bfc09/orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5", size = 128000, upload-time = "2026-08-14T16:12:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c9/97b1ce0112ebf5e949c775ed5b1755e562233179f3584579673cc24d6378/orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a", size = 122106, upload-time = "2026-08-14T16:12:41.324Z" }, + { url = "https://files.pythonhosted.org/packages/a8/6a/facd8b312e4a0d3a7fa978c7e15821f74a336adf1d65529faec33b48e18b/orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d", size = 126869, upload-time = "2026-08-14T16:12:42.651Z" }, + { url = "https://files.pythonhosted.org/packages/54/cb/d7b78218a987eb8a8ce4eeae0286b1bb679333eb631ea0eeaf6371680bfc/orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900", size = 223397, upload-time = "2026-08-14T16:12:44.003Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/bc87c45e7ec639d35ebefd62618e01939531ac8e171426606a01bda05914/orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03", size = 123662, upload-time = "2026-08-14T16:12:45.433Z" }, + { url = "https://files.pythonhosted.org/packages/94/ee/c9a4ff3f2dbedbbe9e635d0fa72c8866adede09b6335ef9644f53752f0d8/orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8", size = 113374, upload-time = "2026-08-14T16:12:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/75/09/3f330a026a796c8b4c97a6f429652a5e912e7065039bf96ed25e42aa7b25/orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94", size = 130029, upload-time = "2026-08-14T16:12:48.06Z" }, + { url = "https://files.pythonhosted.org/packages/7d/40/094cc53126a3d22f76cdf83b6ea67338bed01d774037621a785aa8e6e5ea/orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806", size = 130528, upload-time = "2026-08-14T16:12:49.362Z" }, + { url = "https://files.pythonhosted.org/packages/bc/74/89bb236deb9565f99434b13052bb40ddfcce4adf3afbfa3132ee7e421468/orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df", size = 131075, upload-time = "2026-08-14T16:12:50.692Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ac/1176360d762c01b5bd34acd56fc098e936c491363d8b6b397ad4aa475547/orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978", size = 135321, upload-time = "2026-08-14T16:12:52.114Z" }, + { url = "https://files.pythonhosted.org/packages/7a/02/bbd881c8b9276d50b998de38b4e97de8ace1aac940b0ee545aedbf65ed00/orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222", size = 127472, upload-time = "2026-08-14T16:12:53.517Z" }, + { url = "https://files.pythonhosted.org/packages/8e/02/a0934d7503e6dcbedd6afac3e7f3f8597fd09389949ad94d0f7540e9dbca/orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1", size = 128000, upload-time = "2026-08-14T16:12:55.14Z" }, + { url = "https://files.pythonhosted.org/packages/52/87/69f98f8d40faff103a965a5fbb83f08241b01beaf92badb5413fbc9358cc/orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2", size = 121841, upload-time = "2026-08-14T16:12:56.507Z" }, + { url = "https://files.pythonhosted.org/packages/e6/07/b83046a4e3cadcc0987d0f160696107c4af706a619b56e4ad01940cadadf/orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e", size = 126765, upload-time = "2026-08-14T16:12:57.806Z" }, + { url = "https://files.pythonhosted.org/packages/12/9d/3931253e6f3148abf2cbe14830367042a4806b362ea520df2303db188fb9/orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d", size = 223391, upload-time = "2026-08-14T16:12:59.184Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/b4a4f1e305367245877b967a0bad70fcf001d77c54ac4339a120b66fdae4/orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647", size = 123659, upload-time = "2026-08-14T16:13:00.548Z" }, + { url = "https://files.pythonhosted.org/packages/96/f3/6782c6fa85e2702bc66be183c3b421486167dcf266ee4dc1403fe3824870/orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c", size = 113337, upload-time = "2026-08-14T16:13:02.009Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/b32ab64bacda9d0fa4942ef483bd03cabf0eaf2be819ca9fb7ff610c559d/orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc", size = 130112, upload-time = "2026-08-14T16:13:03.404Z" }, + { url = "https://files.pythonhosted.org/packages/ee/49/6e6142999ca01509219be5e5a9c338a3e5ea011f63e91ff473fbbf3734ed/orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1", size = 130520, upload-time = "2026-08-14T16:13:04.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/d0/3745af0a4cc9867784f29722929cec4d10bd1c877cd754b01ba6d96eb21a/orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a", size = 131053, upload-time = "2026-08-14T16:13:06.14Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f4/6fe5a22fa478fffb190e65c338c84df5c311ef597b363150a17cc57063c0/orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e", size = 135321, upload-time = "2026-08-14T16:13:07.544Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/b1b0ec30289646a81a76e2dbaae2686b96fcccb7cb0323dc1dd78cbc7875/orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f", size = 127485, upload-time = "2026-08-14T16:13:08.88Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2b/277404bdcc21c93b112b963655b76443ebfe828f8a3ff1de7d90f8850eb3/orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92", size = 128048, upload-time = "2026-08-14T16:13:10.305Z" }, + { url = "https://files.pythonhosted.org/packages/41/2b/395b36fa2b4ce7af70b651d715e88f80d884b2c2b14a6b53e84d554fb5f0/orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed", size = 121858, upload-time = "2026-08-14T16:13:11.634Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a3/833e895ff452859eebe75093d26691fe9108f1a7a6a08435d7a5780ea652/orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7", size = 126749, upload-time = "2026-08-14T16:13:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/58/64/99c8947ece10c17176af9aae85c4948f1d109da77440ec14d87239efaf73/orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e", size = 223398, upload-time = "2026-08-14T16:13:14.694Z" }, + { url = "https://files.pythonhosted.org/packages/3e/30/cf983fe09f2731420fda097a9f7ef4343f47fa216c228961ad8f6da44f3d/orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517", size = 123655, upload-time = "2026-08-14T16:13:16.221Z" }, + { url = "https://files.pythonhosted.org/packages/11/50/9cb8ae73fa4749dbbc20f617004213b5ff01c20aaeec34c3f31124f2c1d8/orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38", size = 130515, upload-time = "2026-08-14T16:13:17.601Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0a/adb6ce1a5b5fbf9cb1790f9961bb668a0dd5429aadaf6cee044724681795/orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d", size = 113327, upload-time = "2026-08-14T16:13:18.927Z" }, + { url = "https://files.pythonhosted.org/packages/51/5c/d17f61581d8dbdde7048f87a330fa24915edec38db4d72b381fec14fbb56/orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13", size = 130105, upload-time = "2026-08-14T16:13:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b7/938befcf33bee4704a92ecec6a2731224c539d939bf9429fd39396d28931/orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328", size = 131049, upload-time = "2026-08-14T16:13:21.719Z" }, + { url = "https://files.pythonhosted.org/packages/b0/15/cfa2021d64d5aa8bb5c9f604ef375e00ec8b657651b5dd650b1b7ad13df1/orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c", size = 135320, upload-time = "2026-08-14T16:13:23.415Z" }, + { url = "https://files.pythonhosted.org/packages/1a/50/3e75dfe357c1e8f9e287c7a5740260ef15bd23a5299eae8d0835dcad5375/orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a", size = 127488, upload-time = "2026-08-14T16:13:24.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/a6/79aed402eb3ab284dc5b4791a7ad62c5875127de01b8e3f04bd92d551298/orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55", size = 128048, upload-time = "2026-08-14T16:13:26.217Z" }, + { url = "https://files.pythonhosted.org/packages/64/f7/2723e264aab7248c1ed6ecaad8e5d0cb866c0cffde75442102ffa7491aba/orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578", size = 121860, upload-time = "2026-08-14T16:13:27.577Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/630c9113ec8996778f1f0304b364b091b9a9db5fef5fdc17cca622f5ea24/orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc", size = 126754, upload-time = "2026-08-14T16:13:28.962Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -1455,6 +1571,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + [[package]] name = "parso" version = "0.8.7" @@ -1878,6 +2040,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/97/03635143a12a5d941f545548b00f8ac39d35565321a2effb4154ed267338/pyinstrument-5.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:b6a71f5e7f53c86c9b476b30cf19509463a63581ef17ddbd8680fee37ae509db", size = 128164, upload-time = "2026-01-04T18:38:32.281Z" }, ] +[[package]] +name = "pymilvus" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "grpcio" }, + { name = "orjson" }, + { name = "pandas" }, + { name = "protobuf" }, + { name = "python-dotenv" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/78/6bd0dba340706bc63346af96f0ebe36ff17f75404f5de935fda94d476c98/pymilvus-3.0.1.tar.gz", hash = "sha256:c02389059088b18d6e598cd175541e445c772fab4926c5e527c4913be34887f1", size = 347593, upload-time = "2026-07-29T14:55:43.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/9d/7011887b29f452905745e8bd321f404068d5bfe78fe84e42c0b7cd81a065/pymilvus-3.0.1-py3-none-any.whl", hash = "sha256:c5a8d5c1fa1de7b416e3529d383d8cc2e7da2170433ffa4a2d9087e14f70171a", size = 386820, upload-time = "2026-07-29T14:55:44.279Z" }, +] + [[package]] name = "pytest" version = "9.0.3"