diff --git a/Cargo.lock b/Cargo.lock index 6460abc0..57c5cbcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,6 +238,7 @@ name = "character_memory" version = "0.1.5" dependencies = [ "async-trait", + "character_memory", "chrono", "config", "dotenvy", diff --git a/Cargo.toml b/Cargo.toml index 01ad1efd..a09f73a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,10 @@ name = "character_memory" version = "0.1.5" edition = "2021" +[features] +default = [] +test-fixtures = [] + [dependencies] # Core config = "0.15.22" @@ -29,6 +33,7 @@ uuid = { version = "1.23.1", features = ["v4", "v5", "serde"] } chrono = { version = "0.4.44", features = ["serde"] } [dev-dependencies] +character_memory = { path = ".", features = ["test-fixtures"] } mockall = "0.14.0" tonic = "0.14.6" tempfile = "3.20.0" diff --git a/docs/design/database/README.md b/docs/design/database/README.md index 8f07a54a..5bcf691e 100644 --- a/docs/design/database/README.md +++ b/docs/design/database/README.md @@ -3,5 +3,5 @@ These documents explain the storage-schema rationale behind the graph-authoritative architecture. - [Database Schema Cheat Sheet](schema_cheat_sheet.md): compact reference for Qdrant payload fields, Oxigraph classes, graph predicates, and cross-store authority. -- [Vector Database Payload Design](vector_payload_design.md): why Qdrant stores candidate-recall payload hints rather than authoritative memory state. +- [Vector Database Payload Design](vector_payload_design.md): why Qdrant uses an exact five-field candidate record while Oxigraph retains memory authority. - [Graph Database Schema Design](graph_schema_design.md): why Oxigraph/RDF stores canonical objects, typed links, provenance, lifecycle state, bounded expansion context, and how derived retrieval stats support selectivity and fanout policy without becoming graph truth. diff --git a/docs/design/database/graph_schema_design.md b/docs/design/database/graph_schema_design.md index ac9f8a4b..85d67f5b 100644 --- a/docs/design/database/graph_schema_design.md +++ b/docs/design/database/graph_schema_design.md @@ -44,7 +44,7 @@ urn:cmem:link: The graph also stores the UUID, object type, graph URI, and schema version as literal properties. -This is redundant by design. The URI is efficient for graph edges, while the literal fields make debugging, migration checks, and cross-store joins easier. Qdrant carries the same `object_id` and `graph_uri` so vector candidates can be joined back to graph truth without guessing. +This is redundant inside the graph representation by design. The URI is efficient for graph edges, while the literal fields make debugging and migration checks easier. Qdrant carries `object_id`; retrieval uses that stable identity to hydrate graph truth without duplicating the graph URI in vector payloads. ## Object Classes @@ -370,20 +370,19 @@ This means a reopened graph store can answer object queries, link queries, prove Qdrant, Oxigraph, and the retrieval stats store share stable object IDs, but they do not share authority. ```text -Qdrant recalls candidates and applies coarse payload filters +Qdrant recalls candidates and filters by canonical object type Stats supplies derived selectivity/fanout inputs Oxigraph verifies existence, relationships, provenance, lifecycle, and context ``` -This is why the vector payload intentionally duplicates some graph-derived hints. Duplication is acceptable for speed as long as retrieval treats those hints as non-authoritative. +The vector payload does not duplicate graph-derived hints. Retrieval joins candidates to Oxigraph by stable object ID, then applies relationships, provenance, lifecycle, currentness, and context from graph authority. Internal reconciliation diagnostics can report cross-store drift: - vector point exists but graph object is missing - graph object exists but vector point is missing -- vector payload `graph_uri` does not match the canonical graph URI -- vector lifecycle/currentness hints disagree with graph authority - vector payload schema version is unsupported +- vector payload identity or surface token is malformed - graph object is missing required provenance - stats counter refers to graph edge/object state that no longer exists - stats health indicates conservative fallback should be used diff --git a/docs/design/database/schema_cheat_sheet.md b/docs/design/database/schema_cheat_sheet.md index 29ef206e..a31c2cea 100644 --- a/docs/design/database/schema_cheat_sheet.md +++ b/docs/design/database/schema_cheat_sheet.md @@ -9,7 +9,7 @@ This is the compact schema reference. The companion design notes explain why the | Store | Role | Authoritative For | Not Authoritative For | |---|---|---|---| -| Qdrant | Vector candidate recall and coarse payload filtering | Vector points, embedding surfaces, payload hints | Memory existence, relationships, provenance, lifecycle, currentness, entity selectivity | +| Qdrant | Vector candidate recall and object-type prefiltering | Vector points and embedding-surface provenance | Memory content, existence, relationships, provenance, lifecycle, currentness, entity selectivity | | Oxigraph | Graph authority | Memory objects, typed links, provenance, lifecycle, currentness, expansion context | Semantic nearest-neighbor ranking, derived selectivity counters | | RetrievalStatsStore | Derived retrieval-policy statistics | Entity/relation counters, global counters, selectivity inputs, fanout diagnostics | Memory existence, relationships, provenance, lifecycle, currentness, semantic ranking | | Raw store / caller storage | Source material | Raw transcript or source content behind `raw_ref` | Canonical memory state | @@ -27,7 +27,7 @@ Oxigraph decides. | Field | Stored In | Purpose | |---|---|---| | `object_id` / `objectId` | Qdrant payload and graph literal | Stable object UUID | -| `graph_uri` / `graphUri` | Qdrant payload and graph literal | Stable graph resource pointer | +| `graphUri` | Graph literal | Stable graph resource pointer | | `schema_version` / `schemaVersion` | Qdrant payload and graph literal | Persistence and migration marker | Graph URI pattern: @@ -45,63 +45,15 @@ Retrieval stats store keys refer to the same `object_id` / entity ID values, but ## Qdrant Payload Fields -### Identity And Surface - | Field | Type / Shape | Notes | |---|---|---| -| `object_id` | keyword UUID string | Stable vector-to-graph join id | -| `graph_uri` | keyword URI string | Stable graph resource pointer | -| `object_type` | keyword enum | Canonical memory object type | -| `record_type` | keyword enum | Indexed vector record kind | -| `schema_version` | keyword string | Payload migration marker | +| `object_id` | indexed keyword UUID string | Stable vector-to-graph join id | +| `object_type` | indexed keyword enum | Canonical memory object type | | `surface` | keyword enum | Embedded semantic surface | -| `embedding_text` | string | Text used to generate the vector | -| `content_text` | string | Compact readable/debug text | - -### Object-Specific Hints - -| Field | Type / Shape | Notes | -|---|---|---| -| `derived_type` | keyword enum | Derived memory subtype | -| `entity_type` | keyword enum | Entity subtype | -| `thread_status` | keyword enum | Thread lifecycle/status hint | -| `modality` | keyword enum | Source modality | -| `source_conversation_id` | keyword string | Source conversation filter | -| `canonical_key` | keyword string | Stable caller/domain key | - -### Relationship Hints - -Relationship fields in Qdrant are filter hints only. Oxigraph remains authoritative. +| `schema_version` | keyword string | Record compatibility marker | +| `embedding_text` | text | Exact text used to generate the vector; not read-out content | -| Field | Type / Shape | Notes | -|---|---|---| -| `episode_ids` | keyword array | Related episode ids | -| `observation_ids` | keyword array | Related observation ids | -| `thread_ids` | keyword array | Related thread ids | -| `entity_ids` | keyword array | Related entity ids | -| `participant_entity_ids` | keyword array | Episode participant ids | -| `speaker_entity_id` | keyword UUID string | Observation speaker id | -| `supersedes` | keyword array | Supersession hint | - -Do not compute entity selectivity from Qdrant relationship hints. Qdrant may identify candidate entities from vector hits, but selectivity counts must come from graph-authoritative writes or graph-derived stats. - -### Lifecycle, Ranking, And Time Hints - -| Field | Type / Shape | Notes | -|---|---|---| -| `retention_state` | keyword enum | Lifecycle filter hint | -| `is_current` | bool | Currentness hint | -| `is_superseded` | bool | Supersession/currentness hint | -| `salience_score` | float | Ranking/filter hint | -| `confidence` | float | Ranking/filter hint | -| `stability` | keyword enum | Derived memory stability | -| `created_at` | datetime | Creation time | -| `updated_at` | datetime | Update time | -| `started_at` | datetime | Episode start time | -| `ended_at` | datetime | Episode end time | -| `observed_at` | datetime | Observation time | -| `last_touched_at` | datetime | Thread recency | -| `raw_ref` | keyword string | Source pointer, not raw transcript content | +These are the only Qdrant payload fields. Readable content, graph URI, object-specific state, relationships, lifecycle/currentness, ranking, timestamps, provenance, and raw references are hydrated from Oxigraph by `object_id`. Existing obsolete extra fields may remain on old points but readers ignore them. ## Qdrant Indexed Object Types @@ -264,4 +216,4 @@ The final context pack follows Oxigraph state. ## Reconciliation Diagnostics -Internal diagnostics can report vector-only records, graph-only records, graph URI mismatch, stale lifecycle/currentness hints, unsupported vector schema versions, graph records with missing required provenance, stats records missing graph authority, stats health failures, and low-selectivity expansions rejected by policy. The initial boundary is report-only; diagnostics do not repair stores or expose a public facade API by default. +Internal diagnostics can report vector-only records, graph-only records, unsupported vector schema versions, malformed vector identity or surface tokens, graph records with missing required provenance, stats records missing graph authority, stats health failures, and low-selectivity expansions rejected by policy. The initial boundary is report-only; diagnostics do not repair stores or expose a public facade API by default. diff --git a/docs/design/database/vector_payload_design.md b/docs/design/database/vector_payload_design.md index 9c00ba95..de566b45 100644 --- a/docs/design/database/vector_payload_design.md +++ b/docs/design/database/vector_payload_design.md @@ -1,226 +1,89 @@ # Vector Database Payload Design -> Supersession note (2026-09-02): ADR-I-0025 replaced the payload field inventory below with a five-field read contract (object id, object type, surface, schema version, embedded text) shared by the service and embedded adapters, and ADR-I-0024 rules what any returning prefilter column must satisfy (synchronised or immutable, unknown never matches). The Record Shape, Payload Categories, Indexing Policy, and relationship, lifecycle, time, and text-surface sections remain as the dated design rationale they were (Record Shape still lists the graph URI, which the read contract dropped); the Design Goal (except its third rule, which named relationship and lifecycle hints the read contract dropped), Why Natural-Language Surfaces, and Consistency Model sections remain current. +> Current contract: [ADR-I-0025](../../decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md) supersedes the former denormalized payload-hint inventory with the five-field read contract documented here. [ADR-I-0024](../../decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md) governs any future prefilter re-entry. -This document describes the Qdrant payload design for Character Memory. It is intentionally a design note, not a field-by-field copy of the Rust mapping code. +This document describes the Qdrant record contract for Character Memory. Qdrant is the semantic candidate index, while Oxigraph is the authority for memory content, relationships, provenance, lifecycle state, and currentness. -Qdrant is the semantic candidate index. It is not the memory database of record. The authoritative memory state lives in the graph store. A Qdrant hit means "this object may be relevant"; it does not mean "this object is current, related, or safe to include." Retrieval must verify candidates through the graph authority before returning them in a continuity context pack. +A Qdrant hit means that an object may be relevant. Retrieval must hydrate and verify that object through graph authority before it can enter a continuity context pack. -## Design Goal +## Record Contract -The vector payload exists to make candidate recall cheap without duplicating the graph model. +Each Qdrant point carries exactly five payload fields: -That leads to three rules: +| Field | Shape | Purpose | +|---|---|---| +| object_id | UUID keyword | Stable vector-to-graph join identity | +| object_type | closed keyword enum | Canonical memory object kind | +| surface | closed keyword enum | Semantic surface represented by the vector | +| schema_version | keyword string | Record compatibility marker | +| embedding_text | text | Exact natural-language input used to create the vector | -1. Store natural-language embedding surfaces in vectors. -2. Store only enough metadata to prefilter and join candidates. -3. Treat relationship and lifecycle payload fields as hints until Oxigraph verifies them. +The service indexes only object_id and object_type. The other three fields are stored provenance, not prefilter columns. -This split is deliberate. A vector database is excellent at finding nearby text and applying coarse filters. It is not the right place to decide graph truth, supersession, provenance, lifecycle policy, entity selectivity, or fanout policy. +The record intentionally excludes readable result text, graph URIs, relationships, provenance links, lifecycle/currentness values, ranking values, timestamps, and raw references. Retrieval obtains those values from the graph-authoritative object identified by object_id. -## Record Shape +## Indexed Object Types And Surfaces -Each indexed Qdrant point represents one vector surface for one canonical memory object. The point id may be adapter-specific, so the stable identity is carried in payload: +The vector-indexed object kinds are: -```text -object_id stable UUID for the memory object -graph_uri deterministic graph URI for joining to Oxigraph -surface semantic surface that was embedded -``` - -The same object can have more than one surface over time, but the mapping keeps the join key stable and explicit. This lets retrieval collect vector candidates, deduplicate by object identity, and ask the graph store for the authoritative object. - -Indexed object types are: - -```text +~~~text episode observation entity memory_thread derived_memory -``` +~~~ + +memory_link remains graph-authoritative relationship data and has no embedding surface. -`memory_link` is graph-authoritative relationship data and is not indexed as a semantic memory object by default. +The public maximum-surfaces policy is colocated with the builders. This release emits at most one surface for each vector-indexed object and zero for memory_link. Publishing that limit lets callers bound recall expansion without guessing from implementation details. -## Why Natural-Language Surfaces +## Natural-Language Embedding Text -Embedding text should describe the memory in language a model or user might use later. It should not be a serialized metadata template. +Embedding text should describe the memory in language a model or user might use later. It must not serialize record metadata. -Good embedding surface: +Good: -```text +~~~text The user prefers deterministic public facade tests. -``` +~~~ -Poor embedding surface: +Bad: -```text +~~~text object_type=derived_memory; retention_state=active; confidence=0.82 -``` - -The first supports semantic recall. The second teaches the embedding model about field names rather than memory meaning. Metadata belongs in payload filters, not in the embedded text. - -## Payload Categories - -The implemented payload fields fall into a few design categories. - -### Identity And Versioning - -```text -object_id -graph_uri -object_type -record_type -schema_version -surface -``` - -These fields make vector-to-graph joins deterministic and make migration auditable. `object_type` names the canonical memory object, while `record_type` is retained so future vector records can diverge from domain object classes without changing the join contract. - -### Text Surfaces - -```text -embedding_text -content_text -``` - -`embedding_text` is the exact text used to generate the vector. `content_text` is a compact readable payload for debugging, inspection, and possible re-indexing workflows. Neither field is a raw transcript store. - -Raw interaction material should be addressed through `raw_ref` pointers owned by a caller-managed transcript system. Production raw storage is deferred in v0.1 and is not part of the Qdrant payload contract. - -### Object-Specific Filter Hints - -```text -derived_type -entity_type -thread_status -modality -source_conversation_id -canonical_key -``` - -These fields let Qdrant avoid returning obviously irrelevant records. They are not a substitute for domain validation. For example, `thread_status` may help avoid archived threads during candidate recall, but the graph store still decides whether a thread belongs in the final context pack. - -### Relationship Hints - -```text -episode_ids -observation_ids -thread_ids -entity_ids -participant_entity_ids -speaker_entity_id -supersedes -``` - -These are denormalized hints. They exist because filters like "memories about this entity" or "memories in this thread" are common and should not require a large vector search before graph expansion. +~~~ -They are intentionally called hints because relationships are graph facts. If a payload says a memory is connected to an entity but the graph no longer agrees, retrieval must follow the graph. +The first supports semantic recall. The second trains similarity on storage vocabulary rather than memory meaning. -Qdrant payload relationship fields should not be used as the source of entity selectivity truth. They may identify candidate entities from vector hits, but selectivity counts must come from graph-authoritative writes or graph-derived stats. +embedding_text is retained so an operator can audit what produced a vector. It is not read-out content. Prompt-ready content is hydrated from graph authority. -### Lifecycle And Ranking Hints +## Typed Tokens -```text -retention_state -is_current -is_superseded -salience_score -confidence -stability -``` +object_type and surface are closed vocabularies. Their persisted spellings are owned by the domain enums through one Display and one FromStr implementation per enum. Adapters must not maintain independent token tables. -These fields reduce work before graph verification. They also make lifecycle cleanup more visible during operational inspection. +Unknown tokens fail candidate decoding. This prevents a new producer variant from being silently accepted or mapped to the wrong meaning. -They are not sufficient for final inclusion. The design explicitly supports the case where vector cleanup fails after graph mutation: retrieval should still exclude stale graph records even if Qdrant still returns them. +## Consistency And Migration -### Time Hints +Graph writes may succeed while vector maintenance fails. Public outcomes therefore report typed vector-indexing failures, and retrieval always verifies vector candidates against current graph state. -```text -created_at -updated_at -started_at -ended_at -observed_at -last_touched_at -``` +The five-field change does not bump schema_version. Existing points may still contain obsolete extra fields; readers ignore those fields, and new writes emit only the five-field contract. No in-place payload migration is required. A rebuild from graph authority removes old extras naturally. -Time filters support recency and episode/thread constraints without embedding time into the semantic vector. This keeps time an explicit retrieval dimension instead of hoping the embedding captures it. +A future change that alters the meaning or required interpretation of the five fields must use the repository's schema-version policy. Adding graph-derived prefilter columns also requires an explicit consistency design: every write path must synchronize them, or the values must be immutable, and unknown values must never match. -### Source Pointer +## Indexing Admission -```text -raw_ref -``` +The write-side indexing service rejects a zero-norm record embedding before calling the Qdrant adapter. The failure identifies the affected memory object through the public typed indexing-cause contract. This mirrors the query-side rule that cosine search must not receive a zero-norm query. -`raw_ref` preserves a pointer to source material without storing the full raw chat or voice transcript in Qdrant. The pointer is provenance metadata, not raw transcript content, and it does not promise that the vector layer can resolve the source publicly. +## Operational Checks -## Indexing Policy +Useful checks are: -High-value Qdrant payload indexes are the fields that either reduce candidate set size or protect lifecycle correctness: +- graph objects with no vector point +- vector points whose graph object no longer exists +- unsupported record schema versions +- malformed or unknown object_type and surface tokens +- zero-norm embeddings rejected before adapter dispatch -```text -object_id -graph_uri -object_type -record_type -derived_type -entity_type -thread_status -schema_version -retention_state -episode_ids -observation_ids -thread_ids -entity_ids -participant_entity_ids -speaker_entity_id -supersedes -modality -source_conversation_id -canonical_key -created_at -updated_at -started_at -ended_at -observed_at -last_touched_at -is_current -is_superseded -salience_score -confidence -stability -raw_ref -``` - -The intent is not to index every interesting fact. The intent is to index facts that are useful before graph expansion. Rich relationship traversal belongs in the graph store. Selectivity and fanout policy belong in the derived retrieval stats store. - -## Consistency Model - -Writes that affect indexed memory normally update stores in this direction: - -```text -remember graph upsert, then vector upsert, then stats delta -correct graph supersession, then vector delete/upsert maintenance, then stats delta -forget graph lifecycle mutation, then vector delete maintenance, then stats delta -retrieve vector candidates, then stats-guided fanout, then graph verification -``` - -This is not a distributed transaction. The design assumes partial vector maintenance failure can happen. The correctness rule is therefore: - -```text -Graph state wins over Qdrant payload state. -``` - -If Qdrant is stale, retrieval may do extra work, but it should not return stale or suppressed memories as current context after graph verification. - -If stats are missing or unhealthy, retrieval should degrade conservatively rather than expand broadly. - -## Future Revisit Points - -Revisit this design when: - -- multiple vector surfaces per object become common enough to require public surface policy -- spatial/location retrieval becomes a real product requirement -- a belief/claim subsystem introduces new indexable object types -- Qdrant becomes transactionally synchronized with graph authority, which is not the current design +Obsolete extra payload fields are not read and are not treated as authority. diff --git a/src/adapters/qdrant/payload.rs b/src/adapters/qdrant/payload.rs index a6efc18e..55b41d6f 100644 --- a/src/adapters/qdrant/payload.rs +++ b/src/adapters/qdrant/payload.rs @@ -1,19 +1,14 @@ -// Qdrant payload mapping. These fields are denormalized candidate -// recall/filter hints, not graph authority. -use chrono::{DateTime, SecondsFormat, Utc}; +// Qdrant stores only vector identity, provenance, version, and embedding input. +// Read-out content and graph state are hydrated from graph authority storage. use qdrant_client::qdrant::FieldType; -use serde::Serialize; use crate::domain::schema::require_current_schema_version; use crate::errors::CustomError; -use crate::models::vector::{VectorRecord, VectorSurface}; +use crate::models::vector::VectorRecord; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum QdrantPayloadKind { Keyword, - Datetime, - Bool, - Float, Text, } @@ -21,9 +16,6 @@ impl QdrantPayloadKind { pub(crate) const fn field_type(self) -> FieldType { match self { Self::Keyword => FieldType::Keyword, - Self::Datetime => FieldType::Datetime, - Self::Bool => FieldType::Bool, - Self::Float => FieldType::Float, Self::Text => FieldType::Text, } } @@ -32,76 +24,20 @@ impl QdrantPayloadKind { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum QdrantPayloadField { ObjectId, - GraphUri, ObjectType, - DerivedType, - EntityType, - ThreadStatus, - SchemaVersion, Surface, + SchemaVersion, EmbeddingText, - ContentText, - RetentionState, - IsCurrent, - IsSuperseded, - EpisodeIds, - ObservationIds, - ThreadIds, - EntityIds, - ParticipantEntityIds, - SpeakerEntityId, - Supersedes, - Modality, - SourceConversationId, - CanonicalKey, - CreatedAt, - UpdatedAt, - StartedAt, - EndedAt, - ObservedAt, - LastTouchedAt, - SalienceScore, - Confidence, - Stability, - RawRef, } impl QdrantPayloadField { pub(crate) const fn name(self) -> &'static str { match self { Self::ObjectId => "object_id", - Self::GraphUri => "graph_uri", Self::ObjectType => "object_type", - Self::DerivedType => "derived_type", - Self::EntityType => "entity_type", - Self::ThreadStatus => "thread_status", - Self::SchemaVersion => "schema_version", Self::Surface => "surface", + Self::SchemaVersion => "schema_version", Self::EmbeddingText => "embedding_text", - Self::ContentText => "content_text", - Self::RetentionState => "retention_state", - Self::IsCurrent => "is_current", - Self::IsSuperseded => "is_superseded", - Self::EpisodeIds => "episode_ids", - Self::ObservationIds => "observation_ids", - Self::ThreadIds => "thread_ids", - Self::EntityIds => "entity_ids", - Self::ParticipantEntityIds => "participant_entity_ids", - Self::SpeakerEntityId => "speaker_entity_id", - Self::Supersedes => "supersedes", - Self::Modality => "modality", - Self::SourceConversationId => "source_conversation_id", - Self::CanonicalKey => "canonical_key", - Self::CreatedAt => "created_at", - Self::UpdatedAt => "updated_at", - Self::StartedAt => "started_at", - Self::EndedAt => "ended_at", - Self::ObservedAt => "observed_at", - Self::LastTouchedAt => "last_touched_at", - Self::SalienceScore => "salience_score", - Self::Confidence => "confidence", - Self::Stability => "stability", - Self::RawRef => "raw_ref", } } } @@ -122,39 +58,19 @@ impl QdrantPayloadSchema { QdrantPayloadKind::Keyword, true, ), - schema( - QdrantPayloadField::GraphUri, - QdrantPayloadKind::Keyword, - true, - ), schema( QdrantPayloadField::ObjectType, QdrantPayloadKind::Keyword, true, ), schema( - QdrantPayloadField::DerivedType, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::EntityType, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::ThreadStatus, + QdrantPayloadField::Surface, QdrantPayloadKind::Keyword, - true, + false, ), schema( QdrantPayloadField::SchemaVersion, QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::Surface, - QdrantPayloadKind::Keyword, false, ), schema( @@ -162,118 +78,6 @@ impl QdrantPayloadSchema { QdrantPayloadKind::Text, false, ), - schema( - QdrantPayloadField::ContentText, - QdrantPayloadKind::Text, - false, - ), - schema( - QdrantPayloadField::RetentionState, - QdrantPayloadKind::Keyword, - true, - ), - schema(QdrantPayloadField::IsCurrent, QdrantPayloadKind::Bool, true), - schema( - QdrantPayloadField::IsSuperseded, - QdrantPayloadKind::Bool, - true, - ), - schema( - QdrantPayloadField::EpisodeIds, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::ObservationIds, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::ThreadIds, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::EntityIds, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::ParticipantEntityIds, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::SpeakerEntityId, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::Supersedes, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::Modality, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::SourceConversationId, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::CanonicalKey, - QdrantPayloadKind::Keyword, - true, - ), - schema( - QdrantPayloadField::CreatedAt, - QdrantPayloadKind::Datetime, - true, - ), - schema( - QdrantPayloadField::UpdatedAt, - QdrantPayloadKind::Datetime, - true, - ), - schema( - QdrantPayloadField::StartedAt, - QdrantPayloadKind::Datetime, - true, - ), - schema( - QdrantPayloadField::EndedAt, - QdrantPayloadKind::Datetime, - true, - ), - schema( - QdrantPayloadField::ObservedAt, - QdrantPayloadKind::Datetime, - true, - ), - schema( - QdrantPayloadField::LastTouchedAt, - QdrantPayloadKind::Datetime, - true, - ), - schema( - QdrantPayloadField::SalienceScore, - QdrantPayloadKind::Float, - true, - ), - schema( - QdrantPayloadField::Confidence, - QdrantPayloadKind::Float, - true, - ), - schema( - QdrantPayloadField::Stability, - QdrantPayloadKind::Keyword, - true, - ), - schema(QdrantPayloadField::RawRef, QdrantPayloadKind::Keyword, true), ]; pub(crate) fn indexed_fields() -> impl Iterator { @@ -301,64 +105,14 @@ const fn schema( } pub(crate) const OBJECT_ID_FIELD: &str = QdrantPayloadField::ObjectId.name(); -#[cfg(test)] -pub(crate) const GRAPH_URI_FIELD: &str = QdrantPayloadField::GraphUri.name(); pub(crate) const OBJECT_TYPE_FIELD: &str = QdrantPayloadField::ObjectType.name(); -#[cfg(test)] -pub(crate) const DERIVED_TYPE_FIELD: &str = QdrantPayloadField::DerivedType.name(); -#[cfg(test)] -pub(crate) const SCHEMA_VERSION_FIELD: &str = QdrantPayloadField::SchemaVersion.name(); pub(crate) const SURFACE_FIELD: &str = QdrantPayloadField::Surface.name(); -#[cfg(test)] -pub(crate) const RETENTION_STATE_FIELD: &str = QdrantPayloadField::RetentionState.name(); -#[cfg(test)] -pub(crate) const IS_CURRENT_FIELD: &str = QdrantPayloadField::IsCurrent.name(); -#[cfg(test)] -pub(crate) const IS_SUPERSEDED_FIELD: &str = QdrantPayloadField::IsSuperseded.name(); -#[cfg(test)] -pub(crate) const EPISODE_IDS_FIELD: &str = QdrantPayloadField::EpisodeIds.name(); -#[cfg(test)] -pub(crate) const OBSERVATION_IDS_FIELD: &str = QdrantPayloadField::ObservationIds.name(); -#[cfg(test)] -pub(crate) const THREAD_IDS_FIELD: &str = QdrantPayloadField::ThreadIds.name(); -#[cfg(test)] -pub(crate) const ENTITY_IDS_FIELD: &str = QdrantPayloadField::EntityIds.name(); -#[cfg(test)] -pub(crate) const SUPERSEDES_FIELD: &str = QdrantPayloadField::Supersedes.name(); -#[cfg(test)] -pub(crate) const MODALITY_FIELD: &str = QdrantPayloadField::Modality.name(); -#[cfg(test)] -pub(crate) const CREATED_AT_FIELD: &str = QdrantPayloadField::CreatedAt.name(); -#[cfg(test)] -pub(crate) const UPDATED_AT_FIELD: &str = QdrantPayloadField::UpdatedAt.name(); -#[cfg(test)] -pub(crate) const OBSERVED_AT_FIELD: &str = QdrantPayloadField::ObservedAt.name(); -#[cfg(test)] -pub(crate) const LAST_TOUCHED_AT_FIELD: &str = QdrantPayloadField::LastTouchedAt.name(); -#[cfg(test)] -pub(crate) const SALIENCE_SCORE_FIELD: &str = QdrantPayloadField::SalienceScore.name(); -#[cfg(test)] -pub(crate) const CONFIDENCE_FIELD: &str = QdrantPayloadField::Confidence.name(); -#[cfg(test)] -pub(crate) const STABILITY_FIELD: &str = QdrantPayloadField::Stability.name(); -#[cfg(test)] -pub(crate) const RAW_REF_FIELD: &str = QdrantPayloadField::RawRef.name(); - -#[cfg(test)] -pub(crate) const GRAPH_AUTHORITY_NOTE: &str = - "Qdrant relationship ID fields are denormalized filter hints only; GraphAuthorityStore remains authoritative for relationships, provenance, lifecycle, currentness, and graph expansion."; -#[cfg(test)] -pub(crate) const EMBEDDING_TEXT_FIELD: &str = QdrantPayloadField::EmbeddingText.name(); -#[cfg(test)] -pub(crate) const CONTENT_TEXT_FIELD: &str = QdrantPayloadField::ContentText.name(); pub(crate) fn qdrant_payload_map( record: &VectorRecord, ) -> Result, CustomError> { require_current_schema_version(&record.schema_version, "Qdrant payload mapping")?; - let hints = &record.payload_hints; - let relationships = &record.relationship_hints; let mut payload = serde_json::Map::new(); insert_value( &mut payload, @@ -367,490 +121,140 @@ pub(crate) fn qdrant_payload_map( )?; insert_value( &mut payload, - QdrantPayloadField::GraphUri, - record.graph_uri.clone(), + QdrantPayloadField::ObjectType, + record.object_type.to_string(), )?; insert_value( &mut payload, - QdrantPayloadField::ObjectType, - enum_value(record.object_type), + QdrantPayloadField::Surface, + record.surface.to_string(), )?; insert_value( &mut payload, QdrantPayloadField::SchemaVersion, record.schema_version.clone(), )?; - insert_value( - &mut payload, - QdrantPayloadField::Surface, - vector_surface(record.surface), - )?; insert_value( &mut payload, QdrantPayloadField::EmbeddingText, record.embedding_text.clone(), )?; - insert_value( - &mut payload, - QdrantPayloadField::ContentText, - record.content_text.clone(), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::DerivedType, - hints.derived_type.map(enum_value), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::EntityType, - hints.entity_type.map(enum_value), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::ThreadStatus, - hints.thread_status.map(enum_value), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::RetentionState, - record.retention_state.map(enum_value), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::IsCurrent, - record.is_current, - )?; - insert_optional( - &mut payload, - QdrantPayloadField::IsSuperseded, - hints - .is_superseded - .or_else(|| record.is_current.map(|value| !value)), - )?; - insert_non_empty( - &mut payload, - QdrantPayloadField::EpisodeIds, - ids(&relationships.episode_ids), - )?; - insert_non_empty( - &mut payload, - QdrantPayloadField::ObservationIds, - ids(&relationships.observation_ids), - )?; - insert_non_empty( - &mut payload, - QdrantPayloadField::ThreadIds, - ids(&relationships.thread_ids), - )?; - insert_non_empty( - &mut payload, - QdrantPayloadField::EntityIds, - ids(&relationships.entity_ids), - )?; - insert_non_empty( - &mut payload, - QdrantPayloadField::ParticipantEntityIds, - ids(&relationships.participant_entity_ids), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::SpeakerEntityId, - relationships.speaker_entity_id.map(|id| id.to_string()), - )?; - insert_non_empty( - &mut payload, - QdrantPayloadField::Supersedes, - ids(&relationships.supersedes), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::Modality, - hints.modality.map(enum_value), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::SourceConversationId, - hints.source_conversation_id.clone(), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::CanonicalKey, - hints.canonical_key.clone(), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::CreatedAt, - hints.created_at.map(timestamp), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::UpdatedAt, - hints.updated_at.map(timestamp), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::StartedAt, - hints.started_at.map(timestamp), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::EndedAt, - hints.ended_at.map(timestamp), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::ObservedAt, - hints.observed_at.map(timestamp), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::LastTouchedAt, - hints.last_touched_at.map(timestamp), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::SalienceScore, - hints.salience_score, - )?; - insert_optional( - &mut payload, - QdrantPayloadField::Confidence, - hints.confidence, - )?; - insert_optional( - &mut payload, - QdrantPayloadField::Stability, - hints.stability.map(enum_value), - )?; - insert_optional( - &mut payload, - QdrantPayloadField::RawRef, - record.raw_ref.clone(), - )?; Ok(payload) } fn insert_value( payload: &mut serde_json::Map, field: QdrantPayloadField, - value: impl Serialize, + value: impl serde::Serialize, ) -> Result<(), CustomError> { - let field_schema = QdrantPayloadSchema::field_schema(field); - payload.insert( - field_schema.field.name().to_owned(), - serde_json::to_value(value)?, - ); - Ok(()) -} - -fn insert_optional( - payload: &mut serde_json::Map, - field: QdrantPayloadField, - value: Option, -) -> Result<(), CustomError> { - if let Some(value) = value { - insert_value(payload, field, value)?; - } + let schema = QdrantPayloadSchema::field_schema(field); + let previous = payload.insert(field.name().to_owned(), serde_json::to_value(value)?); + debug_assert!(previous.is_none()); + debug_assert_eq!(schema.field, field); Ok(()) } -fn insert_non_empty( - payload: &mut serde_json::Map, - field: QdrantPayloadField, - values: Vec, -) -> Result<(), CustomError> { - if !values.is_empty() { - insert_value(payload, field, values)?; - } - Ok(()) -} - -fn ids(ids: &[uuid::Uuid]) -> Vec { - ids.iter().map(ToString::to_string).collect() -} - -fn timestamp(value: DateTime) -> String { - value.to_rfc3339_opts(SecondsFormat::Secs, true) -} - -fn enum_value(value: impl Serialize) -> String { - serde_json::to_value(value) - .ok() - .and_then(|value| value.as_str().map(ToOwned::to_owned)) - .unwrap_or_default() -} - -fn vector_surface(surface: VectorSurface) -> &'static str { - match surface { - VectorSurface::Summary => "summary", - VectorSurface::Text => "text", - VectorSurface::Name => "name", - VectorSurface::DerivedText => "derived_text", - VectorSurface::Query => "query", - } -} - #[cfg(test)] mod tests { use super::*; - use crate::domain::{ - graph_uri, DerivedType, EntityType, Modality, ObjectType, RetentionState, Stability, - ThreadStatus, DEFAULT_SCHEMA_VERSION, - }; - use crate::models::vector::{ - VectorPayloadHints, VectorRecord, VectorRelationshipHints, VectorSurface, - }; - use chrono::TimeZone; - use qdrant_client::qdrant::FieldType; + use crate::domain::{MemoryId, ObjectType, VectorSurface, DEFAULT_SCHEMA_VERSION}; + use crate::models::vector::VectorRecord; use serde_json::json; - use uuid::Uuid; #[test] - fn payload_maps_identity_text_lifecycle_time_and_scores() { - let object_id = id(40); - let record = derived_memory_record(object_id); + fn payload_maps_exact_five_field_record() { + let object_id = MemoryId::new_v4(); + let record = VectorRecord::new( + object_id, + ObjectType::DerivedMemory, + VectorSurface::DerivedText, + DEFAULT_SCHEMA_VERSION, + "Reflection: Prefer the smallest durable contract.", + ); let payload = qdrant_payload_map(&record).expect("payload maps"); - assert_eq!(payload[OBJECT_ID_FIELD], json!(object_id.to_string())); + assert_eq!(payload.len(), 5); assert_eq!( - payload[GRAPH_URI_FIELD], - json!(graph_uri(ObjectType::DerivedMemory, object_id)) + payload[QdrantPayloadField::ObjectId.name()], + json!(object_id) ); - assert_eq!(payload[OBJECT_TYPE_FIELD], json!("derived_memory")); - assert!(payload.get("record_type").is_none()); - assert_eq!(payload[DERIVED_TYPE_FIELD], json!("reflection")); - assert_eq!(payload[SCHEMA_VERSION_FIELD], json!(DEFAULT_SCHEMA_VERSION)); assert_eq!( - payload[EMBEDDING_TEXT_FIELD], - json!("Reflection: Keep Qdrant filter-only.") + payload[QdrantPayloadField::ObjectType.name()], + json!("derived_memory") ); assert_eq!( - payload[CONTENT_TEXT_FIELD], - json!("Keep Qdrant filter-only.") + payload[QdrantPayloadField::Surface.name()], + json!("derived_text") + ); + assert_eq!( + payload[QdrantPayloadField::SchemaVersion.name()], + json!(DEFAULT_SCHEMA_VERSION) + ); + assert_eq!( + payload[QdrantPayloadField::EmbeddingText.name()], + json!("Reflection: Prefer the smallest durable contract.") ); - assert_eq!(payload[RETENTION_STATE_FIELD], json!("active")); - assert_eq!(payload[IS_CURRENT_FIELD], json!(false)); - assert_eq!(payload[IS_SUPERSEDED_FIELD], json!(true)); - assert_eq!(payload[CREATED_AT_FIELD], json!("2026-04-28T12:00:00Z")); - assert_eq!(payload[UPDATED_AT_FIELD], json!("2026-04-28T12:00:00Z")); - assert_float(&payload[SALIENCE_SCORE_FIELD], 0.91); - assert_float(&payload[CONFIDENCE_FIELD], 0.82); - assert_eq!(payload[STABILITY_FIELD], json!("medium")); - } - - #[test] - fn payload_relationship_ids_are_filter_hints_not_graph_authority() { - let payload = qdrant_payload_map(&derived_memory_record(id(40))).expect("payload maps"); - - assert_eq!(payload[EPISODE_IDS_FIELD], json!([id(10).to_string()])); - assert_eq!(payload[OBSERVATION_IDS_FIELD], json!([id(20).to_string()])); - assert_eq!(payload[THREAD_IDS_FIELD], json!([id(30).to_string()])); - assert_eq!(payload[ENTITY_IDS_FIELD], json!([id(1).to_string()])); - assert_eq!(payload[SUPERSEDES_FIELD], json!([id(99).to_string()])); - assert!(GRAPH_AUTHORITY_NOTE.contains("GraphAuthorityStore")); - assert!(GRAPH_AUTHORITY_NOTE.contains("filter hints")); } #[test] - fn payload_preserves_schema_version_field() { - let payload = qdrant_payload_map(&derived_memory_record(id(40))).expect("payload maps"); - - assert_eq!(payload[SCHEMA_VERSION_FIELD], json!(DEFAULT_SCHEMA_VERSION)); + fn manifest_declares_exact_record_and_only_identity_indexes() { + assert_eq!( + QdrantPayloadSchema::FIELDS, + &[ + schema( + QdrantPayloadField::ObjectId, + QdrantPayloadKind::Keyword, + true + ), + schema( + QdrantPayloadField::ObjectType, + QdrantPayloadKind::Keyword, + true + ), + schema( + QdrantPayloadField::Surface, + QdrantPayloadKind::Keyword, + false + ), + schema( + QdrantPayloadField::SchemaVersion, + QdrantPayloadKind::Keyword, + false + ), + schema( + QdrantPayloadField::EmbeddingText, + QdrantPayloadKind::Text, + false + ), + ] + ); + assert_eq!( + QdrantPayloadSchema::indexed_fields() + .map(|field| field.field) + .collect::>(), + vec![QdrantPayloadField::ObjectId, QdrantPayloadField::ObjectType,] + ); } #[test] - fn payload_preserves_raw_ref_without_full_raw_transcript_field() { + fn payload_rejects_non_current_schema_version() { let record = VectorRecord::new( - id(10), + MemoryId::new_v4(), ObjectType::Episode, - graph_uri(ObjectType::Episode, id(10)), VectorSurface::Summary, - "Episode summary: Short summary.", - "Short summary.", - DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - None, - VectorRelationshipHints::default(), - Some("raw://conversation/chat_123#turn_42".to_owned()), - ); - - let payload = qdrant_payload_map(&record).expect("payload maps"); - - assert_eq!( - payload[RAW_REF_FIELD], - json!("raw://conversation/chat_123#turn_42") + "future_schema", + "Episode summary", ); - assert!(payload.get("raw_transcript").is_none()); - assert!(payload.get("raw_text").is_none()); - assert!(payload.get("transcript").is_none()); - assert!(payload.get("source_transcript").is_none()); - } - - #[test] - fn payload_mapping_rejects_unsupported_schema_versions() { - let mut record = derived_memory_record(id(40)); - record.schema_version = "future_schema".to_owned(); - - let error = qdrant_payload_map(&record).expect_err("unsupported schema fails"); + let error = qdrant_payload_map(&record).expect_err("future schema is rejected"); assert!(matches!( error, CustomError::UnsupportedSchemaVersion { context: "Qdrant payload mapping", - .. - } + expected: DEFAULT_SCHEMA_VERSION, + actual, + } if actual == "future_schema" )); } - - #[test] - fn schema_manifest_drives_payload_keys_and_indexes() { - let payloads = [ - qdrant_payload_map(&derived_memory_record(id(40))).expect("derived payload maps"), - qdrant_payload_map(&fully_populated_record()).expect("complete payload maps"), - ]; - let schema_names = QdrantPayloadSchema::FIELDS - .iter() - .map(|field| field.field.name()) - .collect::>(); - let emitted_names = payloads - .iter() - .flat_map(|payload| payload.keys().map(String::as_str)) - .collect::>(); - let indexed_fields = QdrantPayloadSchema::indexed_fields().collect::>(); - let indexed_names = indexed_fields - .iter() - .map(|field| field.field.name()) - .collect::>(); - - assert_eq!(schema_names.len(), QdrantPayloadSchema::FIELDS.len()); - assert_eq!(emitted_names, schema_names); - assert!(!schema_names.contains("record_type")); - - for expected in [ - OBJECT_TYPE_FIELD, - DERIVED_TYPE_FIELD, - SCHEMA_VERSION_FIELD, - ENTITY_IDS_FIELD, - THREAD_IDS_FIELD, - EPISODE_IDS_FIELD, - MODALITY_FIELD, - CREATED_AT_FIELD, - OBSERVED_AT_FIELD, - LAST_TOUCHED_AT_FIELD, - IS_CURRENT_FIELD, - IS_SUPERSEDED_FIELD, - RETENTION_STATE_FIELD, - SALIENCE_SCORE_FIELD, - CONFIDENCE_FIELD, - RAW_REF_FIELD, - ] { - assert!( - indexed_names.contains(&expected), - "missing index field {expected}" - ); - } - - assert_eq!(field_type(CREATED_AT_FIELD), FieldType::Datetime); - assert_eq!(field_type(IS_CURRENT_FIELD), FieldType::Bool); - assert_eq!(field_type(SALIENCE_SCORE_FIELD), FieldType::Float); - assert!(!indexed_names.contains(&EMBEDDING_TEXT_FIELD)); - assert!(!indexed_names.contains(&CONTENT_TEXT_FIELD)); - } - - fn derived_memory_record(object_id: Uuid) -> VectorRecord { - VectorRecord::new( - object_id, - ObjectType::DerivedMemory, - graph_uri(ObjectType::DerivedMemory, object_id), - VectorSurface::DerivedText, - "Reflection: Keep Qdrant filter-only.", - "Keep Qdrant filter-only.", - DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - Some(false), - VectorRelationshipHints { - episode_ids: vec![id(10)], - observation_ids: vec![id(20)], - thread_ids: vec![id(30)], - entity_ids: vec![id(1)], - supersedes: vec![id(99)], - ..VectorRelationshipHints::default() - }, - None, - ) - .with_payload_hints(VectorPayloadHints { - derived_type: Some(DerivedType::Reflection), - created_at: Some(timestamp_fixture()), - updated_at: Some(timestamp_fixture()), - salience_score: Some(0.91), - confidence: Some(0.82), - stability: Some(Stability::Medium), - is_superseded: Some(true), - ..VectorPayloadHints::default() - }) - } - - fn fully_populated_record() -> VectorRecord { - VectorRecord::new( - id(200), - ObjectType::DerivedMemory, - graph_uri(ObjectType::DerivedMemory, id(200)), - VectorSurface::DerivedText, - "Fully populated embedding text", - "Fully populated content text", - DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - Some(false), - VectorRelationshipHints { - episode_ids: vec![id(201)], - observation_ids: vec![id(202)], - thread_ids: vec![id(203)], - entity_ids: vec![id(204)], - participant_entity_ids: vec![id(205)], - speaker_entity_id: Some(id(206)), - supersedes: vec![id(207)], - }, - Some("raw://fully-populated".to_owned()), - ) - .with_payload_hints(VectorPayloadHints { - modality: Some(Modality::Chat), - derived_type: Some(DerivedType::Reflection), - entity_type: Some(EntityType::Concept), - thread_status: Some(ThreadStatus::Active), - source_conversation_id: Some("conversation-1".to_owned()), - canonical_key: Some("canonical-1".to_owned()), - created_at: Some(timestamp_fixture()), - updated_at: Some(timestamp_fixture()), - started_at: Some(timestamp_fixture()), - ended_at: Some(timestamp_fixture()), - observed_at: Some(timestamp_fixture()), - last_touched_at: Some(timestamp_fixture()), - salience_score: Some(0.91), - confidence: Some(0.82), - stability: Some(Stability::Medium), - is_superseded: Some(true), - }) - } - - fn field_type(name: &str) -> FieldType { - QdrantPayloadSchema::FIELDS - .iter() - .find(|field| field.field.name() == name) - .map(|field| field.kind.field_type()) - .expect("field exists") - } - - fn assert_float(value: &serde_json::Value, expected: f64) { - let actual = value.as_f64().expect("numeric payload value"); - assert!((actual - expected).abs() < 0.000_001); - } - - fn id(value: u128) -> Uuid { - Uuid::from_u128(0x550e_8400_e29b_41d4_a716_4466_5544_0000 + value) - } - - fn timestamp_fixture() -> DateTime { - Utc.with_ymd_and_hms(2026, 4, 28, 12, 0, 0).unwrap() - } } diff --git a/src/adapters/qdrant/store.rs b/src/adapters/qdrant/store.rs index 9554b349..468f93ee 100644 --- a/src/adapters/qdrant/store.rs +++ b/src/adapters/qdrant/store.rs @@ -12,14 +12,12 @@ use qdrant_client::qdrant::{ }; use qdrant_client::{config::QdrantConfig, Qdrant, QdrantError}; -use crate::domain::{MemoryId, ObjectType}; +use crate::domain::MemoryId; use crate::errors::{ CollectionCompatibilityError, CollectionMismatch, CustomError, IoErrorKind, TransportStatus, VectorDatabaseError, VectorDatabaseErrorKind, }; -use crate::models::vector::{ - VectorCandidateMatch, VectorCandidateSearch, VectorRecordEmbedding, VectorSurface, -}; +use crate::models::vector::{VectorCandidateMatch, VectorCandidateSearch, VectorRecordEmbedding}; use crate::ports::vector_candidate::{VectorCandidateRecall, VectorCandidateStore}; use super::payload::{ @@ -495,7 +493,7 @@ fn qdrant_scroll_fetch_limit(fetch_limit: usize) -> Result { fn qdrant_candidate_filter(query: &VectorCandidateSearch) -> Filter { Filter::must([any_field_matches( OBJECT_TYPE_FIELD, - query.object_types.iter().copied().map(object_type_name), + query.object_types.iter().map(ToString::to_string), )]) } @@ -544,8 +542,14 @@ fn qdrant_payload_to_match( CustomError::DatabaseError(format!("Invalid Qdrant object_id payload UUID: {error}")) })?; - let object_type = parse_object_type(payload_string(payload, OBJECT_TYPE_FIELD)?)?; - let surface = parse_vector_surface(payload_string(payload, SURFACE_FIELD)?)?; + let object_type = payload_string(payload, OBJECT_TYPE_FIELD)? + .parse() + .map_err(|error| { + CustomError::DatabaseError(format!("Invalid Qdrant object_type: {error}")) + })?; + let surface = payload_string(payload, SURFACE_FIELD)? + .parse() + .map_err(|error| CustomError::DatabaseError(format!("Invalid Qdrant surface: {error}")))?; Ok(VectorCandidateMatch::new( object_id, @@ -558,13 +562,14 @@ fn qdrant_payload_to_match( fn qdrant_point_id(record: &crate::models::vector::VectorRecord) -> uuid::Uuid { let mut first = 0xcbf29ce484222325_u64; let mut second = 0x9e3779b97f4a7c15_u64; + let surface = record.surface.to_string(); for byte in record .object_id .as_bytes() .iter() .copied() - .chain(surface_name(record.surface).as_bytes().iter().copied()) + .chain(surface.as_bytes().iter().copied()) { first ^= u64::from(byte); first = first.wrapping_mul(0x100000001b3); @@ -592,66 +597,12 @@ fn payload_string( } } -fn object_type_name(object_type: ObjectType) -> &'static str { - match object_type { - ObjectType::Episode => "episode", - ObjectType::Observation => "observation", - ObjectType::Entity => "entity", - ObjectType::MemoryThread => "memory_thread", - ObjectType::DerivedMemory => "derived_memory", - ObjectType::MemoryLink => "memory_link", - } -} - -fn surface_name(surface: VectorSurface) -> &'static str { - match surface { - VectorSurface::Summary => "summary", - VectorSurface::Text => "text", - VectorSurface::Name => "name", - VectorSurface::DerivedText => "derived_text", - VectorSurface::Query => "query", - } -} - -fn parse_object_type(value: String) -> Result { - match value.as_str() { - "episode" => Ok(ObjectType::Episode), - "observation" => Ok(ObjectType::Observation), - "entity" => Ok(ObjectType::Entity), - "memory_thread" => Ok(ObjectType::MemoryThread), - "derived_memory" => Ok(ObjectType::DerivedMemory), - "memory_link" => Ok(ObjectType::MemoryLink), - _ => Err(CustomError::DatabaseError(format!( - "Unknown Qdrant object_type payload value: {value}" - ))), - } -} - -fn parse_vector_surface(value: String) -> Result { - match value.as_str() { - "summary" => Ok(VectorSurface::Summary), - "text" => Ok(VectorSurface::Text), - "name" => Ok(VectorSurface::Name), - "derived_text" => Ok(VectorSurface::DerivedText), - "query" => Ok(VectorSurface::Query), - _ => Err(CustomError::DatabaseError(format!( - "Unknown Qdrant surface payload value: {value}" - ))), - } -} - #[cfg(test)] mod tests { - use super::super::payload::{ - CONTENT_TEXT_FIELD, GRAPH_URI_FIELD, IS_CURRENT_FIELD, RETENTION_STATE_FIELD, - }; use super::*; use crate::api::types::retrieval::VectorRecallCompleteness; - use crate::domain::{graph_uri, RetentionState, DEFAULT_SCHEMA_VERSION}; - use crate::models::vector::{ - CanonicalCandidates, VectorRecord, VectorRecordEmbedding, VectorRelationshipHints, - VectorSurface, - }; + use crate::domain::{ObjectType, VectorSurface, DEFAULT_SCHEMA_VERSION}; + use crate::models::vector::{CanonicalCandidates, VectorRecord, VectorRecordEmbedding}; use qdrant_client::qdrant::condition::ConditionOneOf; use qdrant_client::qdrant::{ point_id::PointIdOptions, value::Kind, vector, vectors, DeleteCollectionBuilder, PointId, @@ -923,15 +874,9 @@ mod tests { VectorRecord::new( object_id, object_type, - graph_uri(object_type, object_id), VectorSurface::Summary, - "Idle-gap regression record", - "Idle-gap regression record", DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - Some(true), - VectorRelationshipHints::default(), - None, + "Idle-gap regression record", ) } @@ -1045,28 +990,16 @@ mod tests { let summary = VectorRecord::new( object_id, ObjectType::Episode, - graph_uri(ObjectType::Episode, object_id), VectorSurface::Summary, - "Episode summary.", - "Episode summary.", DEFAULT_SCHEMA_VERSION, - None, - None, - VectorRelationshipHints::default(), - None, + "Episode summary.", ); let text = VectorRecord::new( object_id, ObjectType::Episode, - graph_uri(ObjectType::Episode, object_id), VectorSurface::Text, - "Episode text.", - "Episode text.", DEFAULT_SCHEMA_VERSION, - None, - None, - VectorRelationshipHints::default(), - None, + "Episode text.", ); let points = qdrant_point_structs(&[ @@ -1087,24 +1020,14 @@ mod tests { } #[test] - fn upsert_points_use_full_vector_record_payloads() { + fn upsert_points_use_exact_five_field_record_payloads() { let object_id = Uuid::new_v4(); - let related_episode_id = Uuid::new_v4(); let record = VectorRecord::new( object_id, ObjectType::DerivedMemory, - graph_uri(ObjectType::DerivedMemory, object_id), VectorSurface::DerivedText, - "Reflection: Qdrant keeps payload details.", - "Qdrant keeps payload details.", DEFAULT_SCHEMA_VERSION, - None, - Some(true), - VectorRelationshipHints { - episode_ids: vec![related_episode_id], - ..VectorRelationshipHints::default() - }, - Some("raw://conversation/chat_123#turn_42".to_owned()), + "Reflection: Qdrant keeps embedding provenance.", ); let points = qdrant_point_structs(&[VectorRecordEmbedding::new(&record, &[0.25, 0.75])]) @@ -1119,16 +1042,8 @@ mod tests { payload_string(&points[0].payload, SURFACE_FIELD).unwrap(), "derived_text" ); - assert_eq!( - payload_string(&points[0].payload, GRAPH_URI_FIELD).unwrap(), - record.graph_uri - ); - assert_eq!( - payload_string(&points[0].payload, CONTENT_TEXT_FIELD).unwrap(), - "Qdrant keeps payload details." - ); - assert!(points[0].payload.contains_key("episode_ids")); - assert!(points[0].payload.contains_key("raw_ref")); + assert_eq!(points[0].payload.len(), 5); + assert!(!points[0].payload.contains_key("graph_uri")); let vector = points[0] .vectors @@ -1171,15 +1086,15 @@ mod tests { } #[test] - fn candidate_mapping_does_not_return_lifecycle_hints_as_authority() { + fn candidate_mapping_ignores_legacy_extra_payload_fields() { let object_id = Uuid::new_v4(); let mut point = scored_point(object_id, ObjectType::DerivedMemory, 0.77); point .payload - .insert(RETENTION_STATE_FIELD.to_owned(), string_value("active")); + .insert("retention_state".to_owned(), string_value("active")); point .payload - .insert(IS_CURRENT_FIELD.to_owned(), bool_value(true)); + .insert("is_current".to_owned(), bool_value(true)); let matched = scored_point_to_match(point).expect("point maps"); @@ -1201,15 +1116,9 @@ mod tests { let record = VectorRecord::new( object_id, ObjectType::DerivedMemory, - graph_uri(ObjectType::DerivedMemory, object_id), VectorSurface::DerivedText, - "Reflection: Qdrant keeps filter hints.", - "Qdrant keeps filter hints.", DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - Some(true), - VectorRelationshipHints::default(), - None, + "Reflection: Qdrant keeps embedding provenance.", ); store.init_collection().await.expect("collection init"); @@ -1301,15 +1210,9 @@ mod tests { VectorRecord::new( *object_id, ObjectType::Episode, - graph_uri(ObjectType::Episode, *object_id), VectorSurface::Summary, - format!("Equal-score episode {object_id}"), - format!("Equal-score episode {object_id}"), DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - Some(true), - VectorRelationshipHints::default(), - None, + format!("Equal-score episode {object_id}"), ) }) .collect::>(); @@ -1362,7 +1265,7 @@ mod tests { ), ( OBJECT_TYPE_FIELD.to_owned(), - string_value(object_type_name(object_type)), + string_value(&object_type.to_string()), ), (SURFACE_FIELD.to_owned(), string_value("derived_text")), ]), diff --git a/src/api/types.rs b/src/api/types.rs index dac11f01..8214ce1f 100644 --- a/src/api/types.rs +++ b/src/api/types.rs @@ -29,7 +29,7 @@ pub use retrieval::{ SectionPressureSummary, SectionScoreComponents, SectionVectorScoreSource, SelectivityCountScope, SelectivityDecision, SelectivityTelemetry, SelectivityTrace, StaleCandidateOmission, StaleCandidateOmissionSummary, StaleCandidateReason, - VectorCandidateTrace, VectorRecallCompleteness, VectorSurface, + VectorCandidateTrace, VectorRecallCompleteness, }; pub use write_plan::{ CandidateCount, CandidateProducerKind, CandidateProvenance, CandidateRationale, CommitOptions, diff --git a/src/api/types/retrieval.rs b/src/api/types/retrieval.rs index 8549d367..9d036a4b 100644 --- a/src/api/types/retrieval.rs +++ b/src/api/types/retrieval.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::domain::{ DerivedMemory, Episode, GraphExpansionBoundedFailureTrace, GraphExpansionBoundedReason, GraphFailureMode, MemoryId, MemoryObjectRef, MemoryThread, ObjectType, Observation, - RelationType, RetentionState, ThreadStatus, + RelationType, RetentionState, ThreadStatus, VectorSurface, }; use crate::errors::{ConfigValidationError, ConfigValidationReason}; @@ -565,16 +565,6 @@ pub enum SectionVectorScoreSource { DerivedFromRoot { root_score: f32 }, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum VectorSurface { - Summary, - Text, - Name, - DerivedText, - Query, -} - #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum RationaleCategory { diff --git a/src/domain.rs b/src/domain.rs index 7350ebc8..bed8879d 100644 --- a/src/domain.rs +++ b/src/domain.rs @@ -18,6 +18,7 @@ pub use write_validation::{ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use std::{fmt, str::FromStr}; use thiserror::Error; pub type MemoryId = uuid::Uuid; @@ -37,6 +38,72 @@ pub enum ObjectType { MemoryLink, } +impl fmt::Display for ObjectType { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Episode => "episode", + Self::Observation => "observation", + Self::Entity => "entity", + Self::MemoryThread => "memory_thread", + Self::DerivedMemory => "derived_memory", + Self::MemoryLink => "memory_link", + }) + } +} + +impl FromStr for ObjectType { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "episode" => Ok(Self::Episode), + "observation" => Ok(Self::Observation), + "entity" => Ok(Self::Entity), + "memory_thread" => Ok(Self::MemoryThread), + "derived_memory" => Ok(Self::DerivedMemory), + "memory_link" => Ok(Self::MemoryLink), + _ => Err(format!("unknown object type token: {value}")), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VectorSurface { + Summary, + Text, + Name, + DerivedText, + Query, +} + +impl fmt::Display for VectorSurface { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Summary => "summary", + Self::Text => "text", + Self::Name => "name", + Self::DerivedText => "derived_text", + Self::Query => "query", + }) + } +} + +impl FromStr for VectorSurface { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "summary" => Ok(Self::Summary), + "text" => Ok(Self::Text), + "name" => Ok(Self::Name), + "derived_text" => Ok(Self::DerivedText), + "query" => Ok(Self::Query), + _ => Err(format!("unknown vector surface token: {value}")), + } + } +} + impl ObjectType { pub const fn graph_segment(self) -> &'static str { match self { @@ -478,3 +545,31 @@ impl MemoryObject { #[cfg(test)] mod tests; + +#[cfg(test)] +mod token_tests { + use super::{ObjectType, VectorSurface}; + + #[test] + fn persisted_object_and_surface_tokens_round_trip() { + for object_type in [ + ObjectType::Episode, + ObjectType::Observation, + ObjectType::Entity, + ObjectType::MemoryThread, + ObjectType::DerivedMemory, + ObjectType::MemoryLink, + ] { + assert_eq!(object_type.to_string().parse(), Ok(object_type)); + } + for surface in [ + VectorSurface::Summary, + VectorSurface::Text, + VectorSurface::Name, + VectorSurface::DerivedText, + VectorSurface::Query, + ] { + assert_eq!(surface.to_string().parse(), Ok(surface)); + } + } +} diff --git a/src/errors.rs b/src/errors.rs index fcd5973a..d58575fa 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -3,7 +3,7 @@ use thiserror::Error; use crate::domain::{ CandidateValidation, GraphExpansionBoundedFailureTrace, LifecycleDtoValidationError, - LifecyclePolicyKnob, MemoryId, ObjectType, + LifecyclePolicyKnob, MemoryId, MemoryObjectRef, ObjectType, }; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -212,6 +212,8 @@ pub enum VectorIndexingCause { Embedding(#[source] EmbeddingError), #[error("embedding cardinality mismatch: expected {expected}, got {actual}")] CardinalityMismatch { expected: usize, actual: usize }, + #[error("zero-norm embedding for {object:?}")] + ZeroNormEmbedding { object: MemoryObjectRef }, #[error("vector database failed: {0}")] VectorDatabase(#[source] VectorDatabaseError), } diff --git a/src/lib.rs b/src/lib.rs index 5222a1a9..1369d6a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,10 @@ pub(crate) mod ports; pub(crate) mod test_support; pub(crate) mod usecases; +#[cfg(feature = "test-fixtures")] +#[doc(hidden)] +pub use crate::models::vector::zero_norm_record_fixture; + // Re-export types for public use pub use crate::api::embedding::EmbeddingProvider; pub use crate::api::types::{ @@ -42,7 +46,6 @@ pub use crate::api::types::{ StatsUpdateStatus, SupersededByEvidence, SuppressionPolicy, VectorCandidateTrace, VectorIndexCandidate, VectorIndexingFailure, VectorMaintenanceFailure, VectorMaintenanceFailureItem, VectorMaintenanceOperation, VectorRecallCompleteness, - VectorSurface, }; pub use crate::config::{ GraphStoreMode, RetrievalStatsHealthFailMode, RetrievalStatsStoreMode, Settings, @@ -55,7 +58,7 @@ pub use crate::domain::{ GraphExpansionBoundedReason, GraphFailureMode, LifecycleDtoValidationError, LifecyclePolicyKnob, MemoryCandidateKind, MemoryId, MemoryLink, MemoryLinkEndpoint, MemoryObject, MemoryObjectRef, MemoryThread, Modality, ObjectType, Observation, - PlanIdentityField, RelationType, RetentionState, Stability, ThreadStatus, + PlanIdentityField, RelationType, RetentionState, Stability, ThreadStatus, VectorSurface, CURRENT_SCHEMA_VERSION, DEFAULT_SCHEMA_VERSION, EPISODIC_MEMORY_SCHEMA_VERSION, }; pub use crate::errors::{ @@ -66,4 +69,5 @@ pub use crate::errors::{ VectorDatabaseError, VectorDatabaseErrorKind, VectorIndexingCause, }; pub use crate::memory::CharacterMemory; +pub use crate::policy::embedding_surface::max_embedding_surfaces; pub use crate::usecases::write_planning::{PreparedCandidateRefs, RememberPlanDefaults}; diff --git a/src/models/vector.rs b/src/models/vector.rs index 26968e96..560c152f 100644 --- a/src/models/vector.rs +++ b/src/models/vector.rs @@ -2,12 +2,32 @@ mod candidate_record; mod embedding_model; mod record; +#[cfg(test)] +pub(crate) use crate::domain::VectorSurface; +#[cfg(any(test, feature = "test-fixtures"))] +use crate::domain::{MemoryId, MemoryObjectRef, ObjectType, DEFAULT_SCHEMA_VERSION}; #[cfg(test)] pub(crate) use candidate_record::VectorCandidateRecord; pub(crate) use candidate_record::{ - CanonicalCandidates, EmbeddingInput, VectorCandidateMatch, VectorCandidateSearch, VectorSurface, + CanonicalCandidates, EmbeddingInput, VectorCandidateMatch, VectorCandidateSearch, }; pub(crate) use embedding_model::EmbeddingModel; -pub(crate) use record::{ - VectorPayloadHints, VectorRecord, VectorRecordEmbedding, VectorRelationshipHints, -}; +pub(crate) use record::{VectorRecord, VectorRecordEmbedding}; + +#[cfg(any(test, feature = "test-fixtures"))] +#[doc(hidden)] +pub fn zero_norm_record_fixture() -> ( + MemoryObjectRef, + crate::domain::VectorSurface, + &'static str, + &'static str, + Vec, +) { + ( + MemoryObjectRef::new(ObjectType::Episode, MemoryId::from_u128(1)), + crate::domain::VectorSurface::Summary, + DEFAULT_SCHEMA_VERSION, + "Episode summary", + vec![0.0, 0.0], + ) +} diff --git a/src/models/vector/candidate_record.rs b/src/models/vector/candidate_record.rs index 2ab0e423..bf78bebc 100644 --- a/src/models/vector/candidate_record.rs +++ b/src/models/vector/candidate_record.rs @@ -2,16 +2,7 @@ // adapters while deterministic tests use narrower subsets. use std::collections::{hash_map::Entry, HashMap}; -use crate::domain::{MemoryId, ObjectType}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum VectorSurface { - Summary, - Text, - Name, - DerivedText, - Query, -} +use crate::domain::{MemoryId, ObjectType, VectorSurface}; #[derive(Debug, Clone, PartialEq)] pub(crate) struct EmbeddingInput { diff --git a/src/models/vector/record.rs b/src/models/vector/record.rs index 3d528744..638d66c5 100644 --- a/src/models/vector/record.rs +++ b/src/models/vector/record.rs @@ -1,15 +1,10 @@ -// Provider-neutral vector record surface. Payload hints remain -// denormalized recall/filter hints; graph state stays authoritative. -use chrono::{DateTime, Utc}; - -use crate::domain::{ - DerivedType, EntityType, MemoryId, Modality, ObjectType, RetentionState, Stability, - ThreadStatus, -}; +// Provider-neutral five-field vector record. Read-out content and graph state +// remain in graph authority storage. +use crate::domain::{MemoryId, ObjectType, VectorSurface}; +use super::EmbeddingInput; #[cfg(test)] use super::VectorCandidateRecord; -use super::{EmbeddingInput, VectorSurface}; #[derive(Debug, Clone, Copy)] pub(crate) struct VectorRecordEmbedding<'a> { @@ -28,89 +23,32 @@ impl<'a> VectorRecordEmbedding<'a> { } } -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub(crate) struct VectorRelationshipHints { - pub(crate) episode_ids: Vec, - pub(crate) observation_ids: Vec, - pub(crate) thread_ids: Vec, - pub(crate) entity_ids: Vec, - pub(crate) participant_entity_ids: Vec, - pub(crate) speaker_entity_id: Option, - pub(crate) supersedes: Vec, -} - -#[derive(Debug, Clone, PartialEq, Default)] -pub(crate) struct VectorPayloadHints { - pub(crate) modality: Option, - pub(crate) derived_type: Option, - pub(crate) entity_type: Option, - pub(crate) thread_status: Option, - pub(crate) source_conversation_id: Option, - pub(crate) canonical_key: Option, - pub(crate) created_at: Option>, - pub(crate) updated_at: Option>, - pub(crate) started_at: Option>, - pub(crate) ended_at: Option>, - pub(crate) observed_at: Option>, - pub(crate) last_touched_at: Option>, - pub(crate) salience_score: Option, - pub(crate) confidence: Option, - pub(crate) stability: Option, - pub(crate) is_superseded: Option, -} - #[derive(Debug, Clone, PartialEq)] pub(crate) struct VectorRecord { pub(crate) object_id: MemoryId, pub(crate) object_type: ObjectType, - pub(crate) graph_uri: String, pub(crate) surface: VectorSurface, - pub(crate) embedding_text: String, - pub(crate) content_text: String, pub(crate) schema_version: String, - pub(crate) retention_state: Option, - pub(crate) is_current: Option, - pub(crate) relationship_hints: VectorRelationshipHints, - pub(crate) payload_hints: VectorPayloadHints, - pub(crate) raw_ref: Option, + pub(crate) embedding_text: String, } impl VectorRecord { - #[allow(clippy::too_many_arguments)] pub(crate) fn new( object_id: MemoryId, object_type: ObjectType, - graph_uri: impl Into, surface: VectorSurface, - embedding_text: impl Into, - content_text: impl Into, schema_version: impl Into, - retention_state: Option, - is_current: Option, - relationship_hints: VectorRelationshipHints, - raw_ref: Option, + embedding_text: impl Into, ) -> Self { Self { object_id, object_type, - graph_uri: graph_uri.into(), surface, - embedding_text: embedding_text.into(), - content_text: content_text.into(), schema_version: schema_version.into(), - retention_state, - is_current, - relationship_hints, - payload_hints: VectorPayloadHints::default(), - raw_ref, + embedding_text: embedding_text.into(), } } - pub(crate) fn with_payload_hints(mut self, payload_hints: VectorPayloadHints) -> Self { - self.payload_hints = payload_hints; - self - } - pub(crate) fn embedding_input(&self) -> EmbeddingInput { EmbeddingInput::new( Some(self.object_id), @@ -135,7 +73,7 @@ impl From<&VectorRecord> for EmbeddingInput { #[cfg(test)] mod tests { use super::*; - use crate::domain::{graph_uri, DEFAULT_SCHEMA_VERSION}; + use crate::domain::DEFAULT_SCHEMA_VERSION; #[test] fn vector_record_converts_to_embedding_input_without_payload_metadata() { @@ -143,15 +81,9 @@ mod tests { let record = VectorRecord::new( object_id, ObjectType::Episode, - graph_uri(ObjectType::Episode, object_id), VectorSurface::Summary, - "Episode summary: Discussed contract tests.", - "Discussed contract tests.", DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - None, - VectorRelationshipHints::default(), - Some("file:raw/ref.txt".to_owned()), + "Episode summary: Discussed contract tests.", ); let input = record.embedding_input(); @@ -161,9 +93,7 @@ mod tests { assert_eq!(input.surface, VectorSurface::Summary); assert_eq!(input.text, "Episode summary: Discussed contract tests."); assert!(!input.text.contains(&object_id.to_string())); - assert!(!input.text.contains(&record.graph_uri)); assert!(!input.text.contains(DEFAULT_SCHEMA_VERSION)); - assert!(!input.text.contains("file:raw/ref.txt")); } #[test] @@ -172,15 +102,9 @@ mod tests { let record = VectorRecord::new( object_id, ObjectType::Observation, - graph_uri(ObjectType::Observation, object_id), VectorSurface::Text, - "Observation excerpt: Use deterministic fakes.", - "Use deterministic fakes.", DEFAULT_SCHEMA_VERSION, - Some(RetentionState::Active), - None, - VectorRelationshipHints::default(), - None, + "Observation excerpt: Use deterministic fakes.", ); let candidate = record.to_candidate_record(vec![0.1, 0.2]); diff --git a/src/policy/embedding_surface.rs b/src/policy/embedding_surface.rs index f43b7325..c4aa3ab4 100644 --- a/src/policy/embedding_surface.rs +++ b/src/policy/embedding_surface.rs @@ -1,126 +1,63 @@ // Embedding-surface builders for graph objects that participate in vector // candidate recall. use crate::domain::{ - graph_uri, DerivedMemory, Entity, Episode, MemoryObject, MemoryThread, ObjectType, Observation, + DerivedMemory, Entity, Episode, MemoryObject, MemoryThread, ObjectType, Observation, + VectorSurface, }; -use crate::models::vector::{ - VectorPayloadHints, VectorRecord, VectorRelationshipHints, VectorSurface, -}; +use crate::models::vector::VectorRecord; + +pub const fn max_embedding_surfaces(object_type: ObjectType) -> usize { + match object_type { + ObjectType::Episode + | ObjectType::Observation + | ObjectType::Entity + | ObjectType::MemoryThread + | ObjectType::DerivedMemory => 1, + ObjectType::MemoryLink => 0, + } +} pub(crate) fn episode_vector_record(episode: &Episode) -> VectorRecord { VectorRecord::new( episode.id, ObjectType::Episode, - graph_uri(ObjectType::Episode, episode.id), VectorSurface::Summary, - prefixed_text("Episode summary", &episode.summary), - clean_text(&episode.summary), episode.schema_version.clone(), - Some(episode.retention_state), - None, - VectorRelationshipHints { - participant_entity_ids: episode.participant_entity_ids.clone(), - ..VectorRelationshipHints::default() - }, - episode.raw_ref.clone(), + prefixed_text("Episode summary", &episode.summary), ) - .with_payload_hints(VectorPayloadHints { - modality: Some(episode.modality), - source_conversation_id: episode.source_conversation_id.clone(), - created_at: Some(episode.created_at), - started_at: episode.started_at, - ended_at: episode.ended_at, - salience_score: Some(episode.salience_score), - ..VectorPayloadHints::default() - }) } pub(crate) fn observation_vector_record(observation: &Observation) -> VectorRecord { VectorRecord::new( observation.id, ObjectType::Observation, - graph_uri(ObjectType::Observation, observation.id), VectorSurface::Text, - prefixed_text("Observation excerpt", &observation.text), - clean_text(&observation.text), observation.schema_version.clone(), - Some(observation.retention_state), - None, - VectorRelationshipHints { - episode_ids: vec![observation.episode_id], - speaker_entity_id: observation.speaker_entity_id, - ..VectorRelationshipHints::default() - }, - observation.raw_ref.clone(), + prefixed_text("Observation excerpt", &observation.text), ) - .with_payload_hints(VectorPayloadHints { - modality: Some(observation.modality), - created_at: Some(observation.created_at), - observed_at: observation.observed_at, - salience_score: Some(observation.salience_score), - ..VectorPayloadHints::default() - }) } pub(crate) fn derived_memory_vector_record(memory: &DerivedMemory) -> VectorRecord { VectorRecord::new( memory.id, ObjectType::DerivedMemory, - graph_uri(ObjectType::DerivedMemory, memory.id), VectorSurface::DerivedText, - prefixed_text(derived_label(memory), &memory.text), - clean_text(&memory.text), memory.schema_version.clone(), - Some(memory.retention_state), - Some(memory.is_current), - VectorRelationshipHints { - episode_ids: memory.derived_from_episode_ids.clone(), - observation_ids: memory.derived_from_observation_ids.clone(), - thread_ids: memory.thread_ids.clone(), - entity_ids: memory.entity_ids.clone(), - supersedes: memory.supersedes.clone(), - ..VectorRelationshipHints::default() - }, - None, + prefixed_text(derived_label(memory), &memory.text), ) - .with_payload_hints(VectorPayloadHints { - derived_type: Some(memory.derived_type), - created_at: Some(memory.created_at), - updated_at: Some(memory.updated_at), - salience_score: Some(memory.salience_score), - confidence: Some(memory.confidence), - stability: Some(memory.stability), - is_superseded: Some(!memory.is_current), - ..VectorPayloadHints::default() - }) } pub(crate) fn memory_thread_vector_record(thread: &MemoryThread) -> VectorRecord { - let content_text = join_clean([thread.title.as_str(), thread.summary.as_str()]); + let surface_text = join_clean([thread.title.as_str(), thread.summary.as_str()]); VectorRecord::new( thread.id, ObjectType::MemoryThread, - graph_uri(ObjectType::MemoryThread, thread.id), VectorSurface::Summary, - prefixed_text("Thread summary", &content_text), - content_text, thread.schema_version.clone(), - None, - None, - VectorRelationshipHints::default(), - None, + prefixed_text("Thread summary", &surface_text), ) - .with_payload_hints(VectorPayloadHints { - thread_status: Some(thread.status), - canonical_key: thread.canonical_key.clone(), - created_at: Some(thread.created_at), - updated_at: Some(thread.updated_at), - last_touched_at: Some(thread.last_touched_at), - salience_score: Some(thread.salience_score), - ..VectorPayloadHints::default() - }) } pub(crate) fn entity_vector_record(entity: &Entity) -> VectorRecord { @@ -130,28 +67,15 @@ pub(crate) fn entity_vector_record(entity: &Entity) -> VectorRecord { format!("Aliases: {}", entity.aliases.join(", ")) }; let summary = entity.summary.as_deref().unwrap_or_default(); - let content_text = join_clean([entity.name.as_str(), alias_text.as_str(), summary]); + let surface_text = join_clean([entity.name.as_str(), alias_text.as_str(), summary]); VectorRecord::new( entity.id, ObjectType::Entity, - graph_uri(ObjectType::Entity, entity.id), VectorSurface::Name, - prefixed_text("Entity", &content_text), - content_text, entity.schema_version.clone(), - None, - None, - VectorRelationshipHints::default(), - None, + prefixed_text("Entity", &surface_text), ) - .with_payload_hints(VectorPayloadHints { - entity_type: Some(entity.entity_type), - canonical_key: entity.canonical_key.clone(), - created_at: Some(entity.created_at), - updated_at: Some(entity.updated_at), - ..VectorPayloadHints::default() - }) } pub(crate) fn memory_object_vector_record(object: &MemoryObject) -> Option { @@ -213,23 +137,15 @@ mod tests { use uuid::Uuid; #[test] - fn episode_builder_uses_summary_surface_and_preserves_filter_hints() { + fn episode_builder_uses_summary_surface() { let episode = episode_fixture(); let record = episode_vector_record(&episode); assert_eq!(record.object_id, episode.id); assert_eq!(record.object_type, ObjectType::Episode); - assert_eq!(record.graph_uri, graph_uri(ObjectType::Episode, episode.id)); assert_eq!(record.surface, VectorSurface::Summary); assert_eq!(record.embedding_text, "Episode summary: Short summary."); - assert_eq!(record.content_text, "Short summary."); assert_eq!(record.schema_version, DEFAULT_SCHEMA_VERSION); - assert_eq!(record.retention_state, Some(RetentionState::Active)); - assert_eq!( - record.relationship_hints.participant_entity_ids, - vec![id(1)] - ); - assert_eq!(record.raw_ref.as_deref(), Some("raw://episode")); assert_embedding_text_excludes_metadata(&record); } @@ -243,26 +159,16 @@ mod tests { record.embedding_text, "Observation excerpt: Important excerpt." ); - assert_eq!(record.content_text, "Important excerpt."); - assert_eq!(record.relationship_hints.episode_ids, vec![id(10)]); - assert_eq!(record.relationship_hints.speaker_entity_id, Some(id(1))); - assert_eq!(record.raw_ref.as_deref(), Some("raw://observation")); assert_embedding_text_excludes_metadata(&record); } #[test] - fn derived_memory_builder_keeps_currentness_and_relationship_hints_out_of_embedding_text() { + fn derived_memory_builder_keeps_graph_state_out_of_embedding_text() { let derived = derived_memory_fixture(); let record = derived_memory_vector_record(&derived); assert_eq!(record.surface, VectorSurface::DerivedText); assert_eq!(record.embedding_text, "Reflection: Derived insight."); - assert_eq!(record.is_current, Some(false)); - assert_eq!(record.relationship_hints.episode_ids, vec![id(10)]); - assert_eq!(record.relationship_hints.observation_ids, vec![id(20)]); - assert_eq!(record.relationship_hints.thread_ids, vec![id(30)]); - assert_eq!(record.relationship_hints.entity_ids, vec![id(1)]); - assert_eq!(record.relationship_hints.supersedes, vec![id(99)]); assert_embedding_text_excludes_metadata(&record); } @@ -317,11 +223,27 @@ mod tests { ); } + #[test] + fn published_surface_limits_match_current_builders() { + let objects = [ + MemoryObject::Episode(episode_fixture()), + MemoryObject::Observation(observation_fixture()), + MemoryObject::DerivedMemory(derived_memory_fixture()), + MemoryObject::MemoryThread(thread_fixture()), + MemoryObject::Entity(entity_fixture()), + MemoryObject::MemoryLink(link_fixture()), + ]; + + for object in &objects { + let produced = usize::from(memory_object_vector_record(object).is_some()); + assert_eq!(produced, max_embedding_surfaces(object.object_type())); + } + } + fn assert_embedding_text_excludes_metadata(record: &VectorRecord) { assert!(!record .embedding_text .contains(&record.object_id.to_string())); - assert!(!record.embedding_text.contains(&record.graph_uri)); assert!(!record.embedding_text.contains(&record.schema_version)); assert!(!record.embedding_text.contains("raw://")); assert!(!record.embedding_text.contains("Retention")); diff --git a/src/usecases/retrieve.rs b/src/usecases/retrieve.rs index 8732f8a0..e61ca08f 100644 --- a/src/usecases/retrieve.rs +++ b/src/usecases/retrieve.rs @@ -10,17 +10,13 @@ use crate::api::types::{ SectionAssignment, SectionAssignmentReason, SectionPressureSummary, SectionScoreComponents, SectionVectorScoreSource, SelectivityTelemetry, StaleCandidateOmission, StaleCandidateOmissionSummary, StaleCandidateReason, VectorCandidateTrace, - VectorSurface as PublicVectorSurface, }; use crate::domain::{ DerivedMemory, DerivedType, GraphExpansionBoundedReason, GraphFailureMode, MemoryId, - MemoryObject, MemoryObjectRef, ObjectType, RelationType, ThreadStatus, + MemoryObject, MemoryObjectRef, ObjectType, RelationType, ThreadStatus, VectorSurface, }; use crate::errors::CustomError; -use crate::models::vector::{ - EmbeddingInput, VectorCandidateMatch, VectorCandidateSearch, - VectorSurface as InternalVectorSurface, -}; +use crate::models::vector::{EmbeddingInput, VectorCandidateMatch, VectorCandidateSearch}; use crate::policy::graph_expansion::graph_expansion_bounded_failure_trace; use crate::policy::{ selectivity_plan_for_candidate, RetrievalSelectivityPolicy, SelectivityPlan, @@ -236,7 +232,7 @@ where .enumerate() .map(|(index, candidate)| VectorCandidateTrace { object: MemoryObjectRef::new(candidate.object_type, candidate.object_id), - surface: public_vector_surface(candidate.surface), + surface: candidate.surface, score: candidate.score, rank: index + 1, }) @@ -264,7 +260,7 @@ where } _ => context.query_text.trim().to_owned(), }; - let input = EmbeddingInput::new(None, None, InternalVectorSurface::Query, text); + let input = EmbeddingInput::new(None, None, VectorSurface::Query, text); self.embedder.embed(&input).await } } @@ -1267,16 +1263,6 @@ fn section_omission_reason(object: &MemoryObject) -> SectionAssignmentReason { } } -fn public_vector_surface(surface: InternalVectorSurface) -> PublicVectorSurface { - match surface { - InternalVectorSurface::Summary => PublicVectorSurface::Summary, - InternalVectorSurface::Text => PublicVectorSurface::Text, - InternalVectorSurface::Name => PublicVectorSurface::Name, - InternalVectorSurface::DerivedText => PublicVectorSurface::DerivedText, - InternalVectorSurface::Query => PublicVectorSurface::Query, - } -} - fn section_limit( section: ContextPackSection, limits: crate::api::types::ContinuitySectionLimits, @@ -1396,7 +1382,7 @@ mod tests { .await .unwrap(); - assert_eq!(embedder.inputs()[0].surface, InternalVectorSurface::Query); + assert_eq!(embedder.inputs()[0].surface, VectorSurface::Query); assert_eq!(outcome.pack.relevant_episodes[0].id, fixtures.episode.id); assert_eq!( outcome.pack.salient_observations[0].id, @@ -1472,7 +1458,7 @@ mod tests { assert!(trace .vector_candidates .iter() - .all(|candidate| candidate.surface == PublicVectorSurface::Summary)); + .all(|candidate| candidate.surface == VectorSurface::Summary)); assert!(!trace.graph_relations.is_empty()); assert!(trace.graph_relations.iter().all(|relation| fixtures .links() @@ -2593,7 +2579,7 @@ mod tests { .upsert_candidates(&[VectorCandidateRecord::new( fixtures.user_preference.id, ObjectType::DerivedMemory, - InternalVectorSurface::DerivedText, + VectorSurface::DerivedText, vec![1.0, 0.0], )]) .await @@ -2920,12 +2906,7 @@ mod tests { } fn candidate(object_id: MemoryId, object_type: ObjectType, score: f32) -> VectorCandidateMatch { - VectorCandidateMatch::new( - object_id, - object_type, - InternalVectorSurface::Summary, - score, - ) + VectorCandidateMatch::new(object_id, object_type, VectorSurface::Summary, score) } #[derive(Debug)] diff --git a/src/usecases/vector_indexing.rs b/src/usecases/vector_indexing.rs index d9294d74..8a2a26ed 100644 --- a/src/usecases/vector_indexing.rs +++ b/src/usecases/vector_indexing.rs @@ -68,6 +68,17 @@ where )); } + if let Some(index) = embeddings + .iter() + .position(|embedding| embedding.iter().all(|value| *value == 0.0)) + { + let object = objects[index]; + return Ok(failed( + objects, + VectorIndexingCause::ZeroNormEmbedding { object }, + )); + } + let record_embeddings = records .iter() .zip(embeddings.iter()) @@ -99,3 +110,79 @@ fn failed(objects: Vec, cause: VectorIndexingCause) -> VectorIn }), } } + +#[cfg(test)] +mod tests { + use super::*; + use async_trait::async_trait; + + use crate::domain::MemoryId; + use crate::models::vector::{zero_norm_record_fixture, EmbeddingInput, VectorCandidateSearch}; + use crate::ports::vector_candidate::VectorCandidateRecall; + + struct FixedEmbedder(Vec); + + #[async_trait] + impl MemoryEmbedder for FixedEmbedder { + async fn embed(&self, _input: &EmbeddingInput) -> Result, CustomError> { + Ok(self.0.clone()) + } + + async fn embed_batch( + &self, + inputs: &[EmbeddingInput], + ) -> Result>, CustomError> { + Ok(vec![self.0.clone(); inputs.len()]) + } + } + + struct AdapterMustNotRun; + + #[async_trait] + impl VectorCandidateStore for AdapterMustNotRun { + async fn upsert_vector_records( + &self, + _records: &[VectorRecordEmbedding<'_>], + ) -> Result<(), CustomError> { + panic!("zero-norm embeddings must be rejected before the adapter") + } + + async fn search_candidates( + &self, + _query: &VectorCandidateSearch, + ) -> Result { + unreachable!("search is not part of this test") + } + + async fn delete_candidates(&self, _object_ids: &[MemoryId]) -> Result<(), CustomError> { + unreachable!("deletion is not part of this test") + } + } + + #[tokio::test] + async fn zero_norm_record_embedding_is_typed_failure_before_adapter() { + let (object, surface, schema_version, embedding_text, embedding) = + zero_norm_record_fixture(); + let record = VectorRecord::new( + object.id, + object.object_type, + surface, + schema_version, + embedding_text, + ); + let store = AdapterMustNotRun; + let embedder = FixedEmbedder(embedding); + let service = VectorIndexingService::new(&store, &embedder); + + let outcome = service.index(vec![record]).await.expect("typed outcome"); + + assert!(outcome.indexed_objects.is_empty()); + assert_eq!( + outcome.failure, + Some(VectorIndexingFailure { + unindexed_objects: vec![object], + cause: VectorIndexingCause::ZeroNormEmbedding { object }, + }) + ); + } +}