From d098ec907af953dfa51191c6e1af0ce3d9c53b2c Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 07:52:38 +0900 Subject: [PATCH 01/14] refactor: retire vector fake and close v0.1.6 --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 2 +- docs/coding-agent/lessons.md | 69 +++++ .../v0-1-6-embedded-vector-recall-plan.md | 28 +- ...v0_1_6_embedded_vector_candidate_recall.md | 2 +- docs/roadmap/development_roadmap.md | 2 +- src/adapters/qdrant/payload.rs | 84 ++++- src/adapters/qdrant/store.rs | 155 ++++++---- src/adapters/qdrant_edge/mod.rs | 113 ++++--- src/api/types/retrieval.rs | 8 + src/memory.rs | 24 +- src/models/vector.rs | 2 - src/models/vector/candidate_record.rs | 45 +-- src/models/vector/record.rs | 31 -- src/policy.rs | 2 - src/ports/vector_candidate.rs | 8 +- src/test_support.rs | 286 ++---------------- src/usecases/correct_forget.rs | 48 +-- src/usecases/retrieve.rs | 17 +- 20 files changed, 421 insertions(+), 509 deletions(-) rename docs/coding-agent/plans/{active => completed}/v0-1-6-embedded-vector-recall-plan.md (79%) diff --git a/Cargo.lock b/Cargo.lock index f3461d96..4fc58156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -689,7 +689,7 @@ dependencies = [ [[package]] name = "character_memory" -version = "0.1.5" +version = "0.1.6" dependencies = [ "async-trait", "character_memory", diff --git a/Cargo.toml b/Cargo.toml index 9188e3db..eef27bba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "character_memory" -version = "0.1.5" +version = "0.1.6" edition = "2021" [features] diff --git a/README.md b/README.md index a6d38471..b3a0c1ab 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ future interactions become more continuous By default, this uses: - OpenAI for embeddings -- Embedded Qdrant Edge for local vector candidate recall and payload filtering +- Embedded Qdrant Edge for local vector candidate recall with object-type scope filtering - Embedded persistent Oxigraph for graph-authoritative memory objects, relationships, provenance, and lifecycle state ```rust diff --git a/docs/coding-agent/lessons.md b/docs/coding-agent/lessons.md index 51662500..30766499 100644 --- a/docs/coding-agent/lessons.md +++ b/docs/coding-agent/lessons.md @@ -520,3 +520,72 @@ Prevention: Evidence: - Worker Task_4 report of 2026-09-03 (commit d232830) and the orchestrator's clean `git status` check on the main checkout. + +## 2026-09-04 — Include Unit Assertions In Deleted-Helper Call-Site Censuses [tags: validation, refactor, tests, worker] + +Context: +- Plan: `docs/coding-agent/plans/active/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 +- Roles involved: Worker + +Symptom: +- The first focused compile after consolidating Qdrant payload reads failed because service-adapter unit assertions still called the deleted local payload helper and relied on its removed surface-field import. + +Root cause: +- The pre-edit trace found the production callers but did not census the helper's unit-test callers before deleting it. + +Fix applied: +- Route the assertions through a test-only raw string accessor and import the surface field in the test module, then rerun the focused compile. + +Prevention: +- Before deleting or moving a shared helper, run an exact-symbol census across the whole repository, including inline unit-test modules, and resolve every hit in the same patch. +- Residual risk / waiver: none. + +Evidence: +- `cargo test adapters::qdrant::payload::tests --lib` reported all remaining `payload_string` and `SURFACE_FIELD` test references before the correction. + +## 2026-09-04 — Compare Recall Results At The Same Contract Boundary [tags: validation, tests, vector-recall, worker] + +Context: +- Plan: `docs/coding-agent/plans/active/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 +- Roles involved: Worker + +Symptom: +- The first library-suite run failed because the new indexed-exact canary compared 200 raw backend rows with the port's canonical 20-candidate result. + +Root cause: +- The assertion crossed the backend-fetch and port-result boundaries without applying the port's canonicalization and requested limit. + +Fix applied: +- Canonicalize and truncate the indexed exact rows to the request limit before comparing them with the unindexed port result. + +Prevention: +- When a test compares backend rows with a port result, explicitly apply the port's ordering, deduplication, and limit rules before asserting equality. +- Residual risk / waiver: none. + +Evidence: +- `cargo test --lib` passed 393 tests before failing only `indexed_test_configuration_reports_boundary_and_matches_exact_recall` on the 200-row versus 20-row comparison. + +## 2026-09-04 — Export The Endpoint For Ignored Qdrant Unit Tests [tags: validation, qdrant, environment, worker] + +Context: +- Plan: `docs/coding-agent/plans/active/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 +- Roles involved: Worker | Orchestrator + +Symptom: +- The first ignored service-Qdrant run failed three tests with `QDRANT_CONNECTION_STRING is required`, while the idle-channel test passed through its separate setup path. + +Root cause: +- The integration tests load `.env`, but the ignored service-adapter unit tests read `QDRANT_CONNECTION_STRING` directly; setting only `REQUIRE_QDRANT_TESTS=1` was insufficient. + +Fix applied: +- Export the existing `.env` endpoint into the test process together with `REQUIRE_QDRANT_TESTS=1`; all four ignored service-Qdrant tests then passed. + +Prevention: +- The live-gate command for ignored Qdrant unit tests must explicitly export both `QDRANT_CONNECTION_STRING` and `REQUIRE_QDRANT_TESTS=1`; do not assume unit tests load `.env`. +- Residual risk / waiver: none. + +Evidence: +- Corrected ignored run: 4 passed, 0 failed, 396 filtered out; the exclusive Qdrant window was then released. diff --git a/docs/coding-agent/plans/active/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md similarity index 79% rename from docs/coding-agent/plans/active/v0-1-6-embedded-vector-recall-plan.md rename to docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index a6e5df1c..371bcc37 100644 --- a/docs/coding-agent/plans/active/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -1,8 +1,8 @@ # Plan: v0.1.6 Embedded Vector Candidate Recall -- status: approved +- status: done - generated: 2026-09-02 -- last_updated: 2026-09-03 +- last_updated: 2026-09-04 - work_type: mixed ## Goal @@ -175,7 +175,14 @@ - owns: - src/test_support.rs - src/**/tests (fake stores only) - - docs/coding-agent/plans/active/v0-1-6-embedded-vector-recall-plan.md + - src/api/types/retrieval.rs (doc comments only) + - src/ports/vector_candidate.rs (doc comments only) + - src/adapters/qdrant/store.rs, src/adapters/qdrant/payload.rs, src/adapters/qdrant_edge/mod.rs (Task_8 audit dispositions only) + - src/models/vector/candidate_record.rs (header comment only) + - tests/vector_port_contract_tests.rs, tests/support/** (point-id parity fixture and fake-retirement fallout) + - README.md (stale filtering line) + - docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md (closeout status) + - docs/coding-agent/plans/** - docs/coding-agent/lessons.md - docs/roadmap/development_roadmap.md - Cargo.toml @@ -186,6 +193,8 @@ - acceptance: - Zero-hit census for the retired fake and record type. - All five checklist rows cite evidence in the Progress Log. + - Task_8 audit dispositions landed: one point-identity derivation (the v5 derivation) shared by both adapters with a parity assertion and no compat shim; one shared read function for the record contract returning the closed error vocabulary, with the service adapter's stringly database errors for payload and scroll-limit faults removed; the canary pins that an indexed shard searched exactly returns the exhaustive result; the service zero-norm verdict's `scanned` comes from the scope count or the loop invariant is stated; the duplicate vector-config validation is reduced to one or justified in place; the stale header comments and README line are corrected. + - The public completeness type documents all four verdict meanings and the `scanned` and `fetched` counters, stating that a closed boundary verdict is deterministic only for the index-returned prefix, never population-level completeness; the port trait's doc comment states the verdict guarantees (tie closure through the shared loop; exhaustive only on an unindexed shard with a closed cohort; open at the bound) so a future adapter cannot label an indexed prefix exhaustive (deferred from the Task_2 review to avoid re-cascading the stack mid-wave). - validation: - kind: command required: true @@ -238,6 +247,19 @@ Append-only editing rule (applies to both logs below): when appending an entry, - 2026-09-04 Wave 1 done: Task_1 (a633b2e, PR #74) and Task_2 (5b30856, PR #75) approved by the Tier D reviewer; Task_2 needed one revision (checked backend-limit conversion with a BoundaryTieOpen regression at the u32 cap; public re-exports) and carries one post-review fix (dimension check before the zero-norm scroll). PRs stacked on the planning PR as stack #76; merges pending. Wave 2 (Task_3) dispatched on a branch stacked on Task_2. - 2026-09-04 Wave 2 done: Task_3 (36ef8c8, PR #77) approved by the Tier D reviewer after two revisions on the zero-norm rejected-record fixture (final shape: one definition behind the non-default `test-fixtures` feature, enabled for integration tests by a self dev-dependency, doc-hidden). The evaluation repository's exhaustive error-vocabulary conversion is recorded as its re-pin obligation (ADR-I-0023), not a finding. PR appended to stack #76. - 2026-09-04 — Merge shape refined: the stack (planning PR at the bottom, one PR per wave above it) is merged in one go at phase end; GitHub's bottom-up stack order blocks interim merges of wave PRs while the planning PR is a draft, which is the intended shape. Waves proceed by branching from the previous wave's approved tip. +- 2026-09-04 Wave 3 done: Task_4 (01de5d0, PR #78) approved by the Tier D reviewer with no findings after one revision (both-adapter zero-norm parity through the shared fixture; a populated-shard canary leg). During the review the stack's base was found to predate main's Rust 1.97 bump; the planning branch was rebased onto main and the stack cascaded (content-identical by range-diff), so the toolchain pin is 1.97 everywhere. Measured: exhaustive scan 3/11/48 ms at 100/1000/5000 records of 1536 dimensions; release footprint delta about 24.3 MB; no crate features to trim. Stack #76 is #72, #74, #75, #77, #78. +- 2026-09-04 — Task_8 altitude audit (Claude, stack tip 01de5d0): APPROVED; ADR-I-0023/0024/0025/0027 boundaries verified holding by reading; zero hint-field re-entry; zero store-private knowledge exported. + - Findings: MEDIUM point identity derived twice (service in-house hash vs embedded v5 derivation; a shard-to-server sync would double points); MEDIUM record read side implemented twice with divergent error classification (service stringly `DatabaseError` escapes the closed vocabulary); LOW filter convention structurally different but equivalent; LOW service zero-norm `scanned` taken from rows returned; LOW embedded exactness rests on `exact: true`, not on the persisted optimizer config; LOW stale header comments and README line. + - Design-value: every phase addition EARNS-ITS-PLACE except the duplicate vector-config validation (OVERSIZED, minor), the `test-fixtures` feature (OVERSIZED advisory; shape ruled in Wave 2, kept), the in-house point-id hash (DELETE advisory), and the scoring fake (DELETE, already Task_7). + - Disposition: all findings folded into Task_7 (owns and acceptance extended above); the point-identity unification is a behaviour change for existing service collections, acceptable under the Compatibility Policy (rebuild from graph authority), pending the decider's veto. +- 2026-09-04 Task_7 implementation closeout: the deterministic scoring fake, its embedding-bearing `VectorCandidateRecord`, conversion hooks, and fake-only tests were deleted; all former success-path users now open the embedded adapter in a temporary shard fixture, while recording and failure-injection fakes remain. `rg -n "FakeVectorCandidateStore|VectorCandidateRecord|cosine_similarity" src tests` returned zero hits. Both adapters now use the single v5 point-ID derivation and the single typed payload reader in `qdrant/payload.rs`; the service zero-norm path counts its exact filtered scope; the embedded canary compares an indexed `exact: true` search with the exhaustive result; and the retained pre-open vector-config validation is justified because it avoids retrying an incompatible shard. +- 2026-09-04 Deferral-reconfirmation checklist evidence: + 1. Canonical-candidates newtype survival: `rg -n "CanonicalCandidates" src tests` shows the newtype remains the `VectorCandidateRecall.candidates` envelope field and is constructed only at adapter/tie-closure and deliberate test-double boundaries; `canonical_candidates_dedupe_identity_at_highest_score_and_totally_order_ties` remains in the service-free suite. + 2. Dual text columns: `rg -n "content_text|embedding_text|CONTENT_TEXT|EMBEDDING_TEXT" src tests` shows no `content_text` column in this repository and only the required five-field `embedding_text` record/provenance path. The companion repository still has its direct Qdrant `content_text` reader, so its before/after vector-only identity-and-text A/B evidence is explicitly pending in that repository. + 3. Search completeness: `backend_fetch_cap_reports_an_open_boundary_without_allocating_rows`, `retrieval_telemetry_preserves_every_vector_recall_completeness_verdict`, the live service boundary tests, and `service_and_embedded_admit_identical_candidates_in_identical_order` cover open, telemetry, closed, and cross-adapter exhaustive-versus-closed behavior through the shared tie loop. + 4. Hint filter semantics: `rg -n "VectorCandidateFilter|CandidateFilter|match_or_unknown|matches_or_unknown|MatchUnknown" src tests` returned zero hits; the query carries only object-type scope, with its empty-scope behavior and predicate rules pinned by ADR-I-0024 and the port contract tests. + 5. Evaluation baseline capability: this repository exposes singleton-scoped traced candidates, completeness telemetry, and `max_embedding_surfaces`; `rg -n "SearchPointsBuilder|QDRANT_OBJECT_ID_FIELD|QDRANT_OBJECT_TYPE_FIELD|QDRANT_CONTENT_TEXT_FIELD" crates/cmem-eval-adapter-cmem/src/lib.rs` still finds the companion repository's direct Qdrant baseline, so its trace migration, row-level identity/rank A/B diff, and post-switch zero-hit census are explicitly pending in that repository. +- 2026-09-04 Task_7 validation closeout: `cargo fmt --all -- --check` passed; `cargo clippy --all-targets --all-features -- -D warnings` passed on Rust 1.97; bare `cargo test` passed 395 library tests, 31 integration tests, and one doc test with five intentional ignores after the point-identity fixture was added; the live-switch `cargo test` passed the same suite with both service/embedded parity tests executing; and the four ignored service-Qdrant tests passed explicitly under the live endpoint. The package is 0.1.6, the phase and roadmap are finished, and this plan is moved to completed. Independent reviewer approval and stack merge remain Orchestrator gates. ## Decision Log (append-only; re-plans and major discoveries) diff --git a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md index a07e0d75..5f7f1fc8 100644 --- a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md +++ b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md @@ -1,6 +1,6 @@ # v0.1.6 Design: Embedded Vector Candidate Recall -Status: decided 2026-09-02, revised 2026-09-03 (ADR-I-0023 through ADR-I-0027); supersedes the 2026-07 draft of this document; the embedded engine ruling (Qdrant Edge over an in-house scan) was taken the same day on two feasibility spikes. +Status: finished 2026-09-04 (ADR-I-0023 through ADR-I-0027); supersedes the 2026-07 draft of this document. The embedded engine ruling (Qdrant Edge over an in-house scan) was taken on two feasibility spikes; companion-repository evaluation evidence that was not yet produced at closeout remains explicitly pending in that repository. ## Version intent diff --git a/docs/roadmap/development_roadmap.md b/docs/roadmap/development_roadmap.md index 00525f49..d00fccfe 100644 --- a/docs/roadmap/development_roadmap.md +++ b/docs/roadmap/development_roadmap.md @@ -259,7 +259,7 @@ Assisted remember workflows may accept raw or semi-raw input as transient proces | v0.1.3 | Remember intake interfaces and deterministic write planning | Finished. Generation-ready write path with `RememberWritePlan`, memory candidates, validation, deterministic helpers, prepare/validate/commit flow, and shared manual/future-generated commit machinery. | | v0.1.4 | Continuity evaluation harness | Finished. Deterministic long-horizon evaluation harness implemented in the public companion `CharacterMemoryEvals` repository as a development aid, not core library functionality: synthetic interaction fixtures, a minimal example assistant loop, continuity-oriented retrieval-quality metrics, selectivity/fanout measurement, and hub-entity stress scenarios. | | v0.1.5 | Eval-driven v0.1 family closeout | Finished. Ran the evaluation harness across the v0.1 family, dispositioned eleven findings (none critical, none open), fixed deterministic vector admission and write-path warning diagnostics in the library, retained the measured defaults with a recorded basis (ADR-I-0022), adopted embedded persistent Oxigraph as the validated default (ADR-I-0021), and expanded the evaluation suite to 33 scenarios including benchmark-adapted and real-embedding fixtures. Closeout report: [`v0_1_5_closeout_report.md`](v0_1_5_closeout_report.md). | -| v0.1.6 | Embedded vector candidate recall | Planned. An embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) behind the vector port as the default vector mode, shipped at its exact-scan indexing threshold, so zero-infrastructure local deployments are the default and the default test path needs no external service, with the service adapter retained as the explicit service mode; a redesigned port contract that reports recall completeness, forbids prefilters that match unknown values, and stores a five-field record (identity, surface, schema version, and the embedded text kept as provenance); the evaluation repository's vector-only baseline moves onto the retrieval trace. Decisions: ADR-I-0023 through ADR-I-0027. | +| v0.1.6 | Embedded vector candidate recall | Finished 2026-09-04. An embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) is the default vector mode at its exact-scan indexing threshold, so zero-infrastructure local deployments and the default test path need no external service; the service adapter remains the explicit service mode. The redesigned port reports recall completeness, accepts only object-type scope, and stores the five-field record shared by both adapters. Companion-repository evaluation work is tracked there. Decisions: ADR-I-0023 through ADR-I-0027. | | v0.2 | Scoped continuity and reflection | `ContinuityScope`, scoped reflection, relationship state between arbitrary entities, character signals for continuing entities, open-loop/commitment lifecycle, and current continuity views. | | v0.3 | Factual rigor, temporal validity, and entity evolution | Assertions, claims, evidence links, belief assessments, source assessment, temporal validity, entity drift handling, and current-belief views. | | v0.4 | Retrieval observability and governance | Retrieval traces, context subgraphs, validation rules, graph health reports, policy diagnostics, rejected expansion traces, cluster/activation diagnostics, and retention assessment. | diff --git a/src/adapters/qdrant/payload.rs b/src/adapters/qdrant/payload.rs index 55b41d6f..3e3d9819 100644 --- a/src/adapters/qdrant/payload.rs +++ b/src/adapters/qdrant/payload.rs @@ -3,8 +3,9 @@ use qdrant_client::qdrant::FieldType; use crate::domain::schema::require_current_schema_version; -use crate::errors::CustomError; -use crate::models::vector::VectorRecord; +use crate::domain::MemoryId; +use crate::errors::{CustomError, VectorDatabaseError, VectorDatabaseErrorKind}; +use crate::models::vector::{VectorCandidateMatch, VectorRecord}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum QdrantPayloadKind { @@ -106,8 +107,62 @@ const fn schema( pub(crate) const OBJECT_ID_FIELD: &str = QdrantPayloadField::ObjectId.name(); pub(crate) const OBJECT_TYPE_FIELD: &str = QdrantPayloadField::ObjectType.name(); +#[cfg(test)] pub(crate) const SURFACE_FIELD: &str = QdrantPayloadField::Surface.name(); +pub(crate) fn qdrant_point_id(record: &VectorRecord) -> MemoryId { + MemoryId::new_v5(&record.object_id, record.surface.to_string().as_bytes()) +} + +pub(crate) fn read_candidate_match<'a>( + backend: &'static str, + score: f32, + fields: impl Fn(QdrantPayloadField) -> Option<&'a str>, +) -> Result { + let required = |field| { + fields(field).ok_or_else(|| { + payload_deserialization_error( + backend, + format!("missing or invalid string field {}", field.name()), + ) + }) + }; + let object_id = required(QdrantPayloadField::ObjectId)? + .parse() + .map_err(|error| { + payload_deserialization_error(backend, format!("invalid object_id UUID: {error}")) + })?; + let object_type = required(QdrantPayloadField::ObjectType)? + .parse() + .map_err(|error| { + payload_deserialization_error(backend, format!("invalid object_type: {error}")) + })?; + let surface = required(QdrantPayloadField::Surface)? + .parse() + .map_err(|error| { + payload_deserialization_error(backend, format!("invalid surface: {error}")) + })?; + + Ok(VectorCandidateMatch::new( + object_id, + object_type, + surface, + score, + )) +} + +pub(crate) fn payload_deserialization_error( + backend: &'static str, + message: impl Into, +) -> VectorDatabaseError { + VectorDatabaseError::new( + backend, + VectorDatabaseErrorKind::PayloadDeserialization, + None, + message, + ) +} + pub(crate) fn qdrant_payload_map( record: &VectorRecord, ) -> Result, CustomError> { @@ -257,4 +312,29 @@ mod tests { } if actual == "future_schema" )); } + + #[test] + fn point_identity_is_the_surface_namespaced_object_identity() { + let record = VectorRecord::new( + MemoryId::from_u128(7), + ObjectType::Episode, + VectorSurface::Summary, + DEFAULT_SCHEMA_VERSION, + "Episode summary", + ); + + assert_eq!( + qdrant_point_id(&record), + MemoryId::new_v5(&record.object_id, b"summary") + ); + } + + #[test] + fn candidate_reader_uses_the_closed_payload_error_vocabulary() { + let error = + read_candidate_match("qdrant", 1.0, |_| None).expect_err("missing identity must fail"); + + assert_eq!(error.kind, VectorDatabaseErrorKind::PayloadDeserialization); + assert!(error.message.contains(OBJECT_ID_FIELD)); + } } diff --git a/src/adapters/qdrant/store.rs b/src/adapters/qdrant/store.rs index 468f93ee..dbd38f18 100644 --- a/src/adapters/qdrant/store.rs +++ b/src/adapters/qdrant/store.rs @@ -5,7 +5,7 @@ use std::{collections::HashMap, time::Duration}; use async_trait::async_trait; use qdrant_client::qdrant::{ - points_selector::PointsSelectorOneOf, value::Kind, vectors_config, Condition, + points_selector::PointsSelectorOneOf, vectors_config, Condition, CountPointsBuilder, CreateCollectionBuilder, CreateFieldIndexCollectionBuilder, DeletePointsBuilder, Distance, Filter, PointStruct, ScoredPoint, ScrollPointsBuilder, SearchPointsBuilder, UpsertPointsBuilder, VectorParams, VectorsConfig, @@ -21,7 +21,8 @@ use crate::models::vector::{VectorCandidateMatch, VectorCandidateSearch, VectorR use crate::ports::vector_candidate::{VectorCandidateRecall, VectorCandidateStore}; use super::payload::{ - qdrant_payload_map, QdrantPayloadSchema, OBJECT_ID_FIELD, OBJECT_TYPE_FIELD, SURFACE_FIELD, + payload_deserialization_error, qdrant_payload_map, qdrant_point_id, read_candidate_match, + QdrantPayloadSchema, OBJECT_ID_FIELD, OBJECT_TYPE_FIELD, }; use super::tie_closure::close_tie_cohort; @@ -186,6 +187,36 @@ impl QdrantVectorCandidateStore { .map(|point| qdrant_payload_to_match(&point.payload, 0.0)) .collect() } + + async fn scoped_count(&self, query: &VectorCandidateSearch) -> Result { + let request = CountPointsBuilder::new(&self.collection_name) + .filter(qdrant_candidate_filter(query)) + .exact(true) + .build(); + let count = self + .client + .count(request) + .await + .map_err(qdrant_error)? + .result + .ok_or_else(|| { + CustomError::VectorDatabaseError(VectorDatabaseError::new( + "qdrant", + VectorDatabaseErrorKind::Response, + None, + "Qdrant count response was missing result", + )) + })? + .count; + usize::try_from(count).map_err(|_| { + CustomError::VectorDatabaseError(VectorDatabaseError::new( + "qdrant", + VectorDatabaseErrorKind::Conversion, + None, + format!("Qdrant scope count {count} exceeds the platform maximum"), + )) + }) + } } fn validate_collection_vector_config( @@ -285,6 +316,11 @@ impl VectorCandidateStore for QdrantVectorCandidateStore { } else { usize::MAX }; + let scanned = if zero_norm { + Some(self.scoped_count(query).await?) + } else { + None + }; let closed = close_tie_cohort(query.limit, fetch_limit_cap, |fetch_limit| async move { if zero_norm { self.scroll_zero_norm_candidate_batch(query, fetch_limit) @@ -294,7 +330,7 @@ impl VectorCandidateStore for QdrantVectorCandidateStore { } }) .await?; - let completeness = closed.completeness(zero_norm.then_some(closed.fetched)); + let completeness = closed.completeness(scanned); Ok(VectorCandidateRecall { candidates: closed.candidates, completeness, @@ -483,9 +519,12 @@ fn qdrant_candidate_config(url: &str) -> QdrantConfig { fn qdrant_scroll_fetch_limit(fetch_limit: usize) -> Result { u32::try_from(fetch_limit).map_err(|_| { - CustomError::DatabaseError(format!( - "Qdrant scroll limit {fetch_limit} exceeds the backend maximum {}", - u32::MAX + CustomError::VectorDatabaseError(payload_deserialization_error( + "qdrant", + format!( + "Qdrant scroll limit {fetch_limit} exceeds the backend maximum {}", + u32::MAX + ), )) }) } @@ -537,69 +576,22 @@ fn qdrant_payload_to_match( payload: &HashMap, score: f32, ) -> Result { - let object_id = payload_string(payload, OBJECT_ID_FIELD)?; - let object_id = uuid::Uuid::parse_str(&object_id).map_err(|error| { - CustomError::DatabaseError(format!("Invalid Qdrant object_id payload UUID: {error}")) - })?; - - let object_type = payload_string(payload, OBJECT_TYPE_FIELD)? - .parse() - .map_err(|error| { - CustomError::DatabaseError(format!("Invalid Qdrant object_type: {error}")) - })?; - let surface = payload_string(payload, SURFACE_FIELD)? - .parse() - .map_err(|error| CustomError::DatabaseError(format!("Invalid Qdrant surface: {error}")))?; - - Ok(VectorCandidateMatch::new( - object_id, - object_type, - surface, - score, - )) -} - -fn qdrant_point_id(record: &crate::models::vector::VectorRecord) -> uuid::Uuid { - let mut first = 0xcbf29ce484222325_u64; - let mut second = 0x9e3779b97f4a7c15_u64; - let surface = record.surface.to_string(); - - for byte in record - .object_id - .as_bytes() - .iter() - .copied() - .chain(surface.as_bytes().iter().copied()) - { - first ^= u64::from(byte); - first = first.wrapping_mul(0x100000001b3); - second ^= u64::from(byte).wrapping_add(0x9e3779b97f4a7c15); - second = second.rotate_left(5).wrapping_mul(0x517cc1b727220a95); - } - - let mut bytes = [0_u8; 16]; - bytes[..8].copy_from_slice(&first.to_be_bytes()); - bytes[8..].copy_from_slice(&second.to_be_bytes()); - bytes[6] = (bytes[6] & 0x0f) | 0x50; - bytes[8] = (bytes[8] & 0x3f) | 0x80; - uuid::Uuid::from_bytes(bytes) -} - -fn payload_string( - payload: &HashMap, - field: &str, -) -> Result { - match payload.get(field).and_then(|value| value.kind.as_ref()) { - Some(Kind::StringValue(value)) => Ok(value.clone()), - _ => Err(CustomError::DatabaseError(format!( - "Missing or invalid string field in Qdrant payload: {field}" - ))), - } + read_candidate_match("qdrant", score, |field| { + payload + .get(field.name()) + .and_then(|value| value.kind.as_ref()) + .and_then(|kind| match kind { + qdrant_client::qdrant::value::Kind::StringValue(value) => Some(value.as_str()), + _ => None, + }) + }) + .map_err(CustomError::VectorDatabaseError) } #[cfg(test)] mod tests { use super::*; + use crate::adapters::qdrant::payload::SURFACE_FIELD; use crate::api::types::retrieval::VectorRecallCompleteness; use crate::domain::{ObjectType, VectorSurface, DEFAULT_SCHEMA_VERSION}; use crate::models::vector::{CanonicalCandidates, VectorRecord, VectorRecordEmbedding}; @@ -608,6 +600,19 @@ mod tests { point_id::PointIdOptions, value::Kind, vector, vectors, DeleteCollectionBuilder, PointId, Value, VectorParamsMap, }; + + fn payload_string<'a>( + payload: &'a HashMap, + field: &str, + ) -> Option<&'a str> { + payload + .get(field) + .and_then(|value| value.kind.as_ref()) + .and_then(|kind| match kind { + Kind::StringValue(value) => Some(value.as_str()), + _ => None, + }) + } use std::env; use std::time::Instant; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -678,7 +683,13 @@ mod tests { assert_eq!(qdrant_scroll_fetch_limit(backend_max).unwrap(), u32::MAX); if let Some(too_large) = backend_max.checked_add(1) { - assert!(qdrant_scroll_fetch_limit(too_large).is_err()); + assert!(matches!( + qdrant_scroll_fetch_limit(too_large), + Err(CustomError::VectorDatabaseError(VectorDatabaseError { + kind: VectorDatabaseErrorKind::PayloadDeserialization, + .. + })) + )); } } @@ -1009,6 +1020,20 @@ mod tests { .expect("points build"); assert_ne!(points[0].id, points[1].id); + assert_eq!( + points[0] + .id + .as_ref() + .and_then(|id| id.point_id_options.as_ref()), + Some(&PointIdOptions::Uuid(qdrant_point_id(&summary).to_string())) + ); + assert_eq!( + points[1] + .id + .as_ref() + .and_then(|id| id.point_id_options.as_ref()), + Some(&PointIdOptions::Uuid(qdrant_point_id(&text).to_string())) + ); assert_eq!( payload_string(&points[0].payload, OBJECT_ID_FIELD).unwrap(), object_id.to_string() diff --git a/src/adapters/qdrant_edge/mod.rs b/src/adapters/qdrant_edge/mod.rs index ed4a2fe3..160566a6 100644 --- a/src/adapters/qdrant_edge/mod.rs +++ b/src/adapters/qdrant_edge/mod.rs @@ -17,8 +17,8 @@ use serde::{Deserialize, Serialize}; use tokio::sync::{oneshot, Mutex}; use crate::adapters::qdrant::payload::{ - qdrant_payload_map, QdrantPayloadKind, QdrantPayloadSchema, OBJECT_ID_FIELD, OBJECT_TYPE_FIELD, - SURFACE_FIELD, + qdrant_payload_map, qdrant_point_id, read_candidate_match, QdrantPayloadKind, + QdrantPayloadSchema, OBJECT_ID_FIELD, OBJECT_TYPE_FIELD, }; use crate::adapters::qdrant::tie_closure::close_tie_cohort; use crate::domain::{MemoryId, DEFAULT_SCHEMA_VERSION}; @@ -170,6 +170,24 @@ impl QdrantEdgeVectorCandidateStore { receive(receiver).await } + #[cfg(test)] + async fn search_batch_exact( + &self, + query: &VectorCandidateSearch, + fetch_limit: usize, + ) -> Result, CustomError> { + let (reply, receiver) = oneshot::channel(); + self.send(Command::Search { + query_embedding: query.query_embedding.clone(), + object_types: object_type_tokens(query), + limit: fetch_limit, + exact: true, + zero_norm: query.is_zero_norm(), + reply, + })?; + receive(receiver).await + } + async fn scoped_count(&self, query: &VectorCandidateSearch) -> Result { let (reply, receiver) = oneshot::channel(); self.send(Command::Count { @@ -265,13 +283,10 @@ impl QdrantEdgeVectorCandidateStore { } .into()); } - let point_id = MemoryId::new_v5( - &record.record.object_id, - record.record.surface.to_string().as_bytes(), - ) - .to_string() - .parse::() - .expect("UUID text is a valid Qdrant Edge point ID"); + let point_id = qdrant_point_id(record.record) + .to_string() + .parse::() + .expect("UUID text is a valid Qdrant Edge point ID"); let payload = serde_json::Value::Object(qdrant_payload_map(record.record)?); Ok(PointStruct::new(point_id, record.embedding.to_vec(), payload).into()) } @@ -363,6 +378,9 @@ fn open_shard( let existing = path.join(EDGE_CONFIG_FILE).is_file(); if existing { validate_marker(&path, collection_name)?; + // Validate before the retry loop so an incompatible shard fails once + // with the typed compatibility error instead of retrying a load that + // cannot succeed. The post-load check also protects newly created shards. let config = EdgeConfig::load(&path) .expect("existing config path must produce a load result") .map_err(edge_error)?; @@ -537,33 +555,21 @@ fn payload_to_match( payload: Option<&qdrant_edge::Payload>, score: f32, ) -> Result { - let payload = payload.ok_or_else(|| payload_error("payload is missing"))?; - let object_id = payload_string(payload, OBJECT_ID_FIELD)? - .parse() - .map_err(|error| payload_error(format!("invalid object_id UUID: {error}")))?; - let object_type = payload_string(payload, OBJECT_TYPE_FIELD)? - .parse() - .map_err(|error| payload_error(format!("invalid object_type: {error}")))?; - let surface = payload_string(payload, SURFACE_FIELD)? - .parse() - .map_err(|error| payload_error(format!("invalid surface: {error}")))?; - Ok(VectorCandidateMatch::new( - object_id, - object_type, - surface, - score, - )) -} - -fn payload_string<'a>( - payload: &'a qdrant_edge::Payload, - field: &str, -) -> Result<&'a str, CustomError> { - payload - .0 - .get(field) - .and_then(serde_json::Value::as_str) - .ok_or_else(|| payload_error(format!("missing or invalid string field {field}"))) + let payload = payload.ok_or_else(|| { + CustomError::VectorDatabaseError( + crate::adapters::qdrant::payload::payload_deserialization_error( + "qdrant_edge", + "payload is missing", + ), + ) + })?; + read_candidate_match("qdrant_edge", score, |field| { + payload + .0 + .get(field.name()) + .and_then(serde_json::Value::as_str) + }) + .map_err(CustomError::VectorDatabaseError) } fn string_filter(field: &str, values: Vec) -> Filter { @@ -690,15 +696,6 @@ fn owner_unavailable() -> CustomError { )) } -fn payload_error(message: impl Into) -> CustomError { - CustomError::VectorDatabaseError(VectorDatabaseError::new( - "qdrant_edge", - VectorDatabaseErrorKind::PayloadDeserialization, - None, - message, - )) -} - fn edge_error(error: impl std::fmt::Display) -> CustomError { CustomError::VectorDatabaseError(VectorDatabaseError::new( "qdrant_edge", @@ -759,6 +756,27 @@ mod tests { (records, embeddings) } + #[tokio::test] + async fn point_identity_matches_the_shared_service_derivation() { + let temp = TempDir::new().unwrap(); + let store = QdrantEdgeVectorCandidateStore::open(temp.path(), "point_identity", 2) + .await + .unwrap(); + let record = VectorRecord::new( + MemoryId::from_u128(7), + ObjectType::Episode, + VectorSurface::Summary, + DEFAULT_SCHEMA_VERSION, + "Episode summary", + ); + + let point = store + .point(&VectorRecordEmbedding::new(&record, &[1.0, 0.0])) + .unwrap(); + + assert_eq!(point.id.to_string(), qdrant_point_id(&record).to_string()); + } + async fn upsert( store: &QdrantEdgeVectorCandidateStore, records: &[VectorRecord], @@ -1109,7 +1127,12 @@ mod tests { let exact_result = exact.search_candidates(&query(20)).await.unwrap(); let indexed_result = indexed.search_candidates(&query(20)).await.unwrap(); + let indexed_exact_result = indexed.search_batch_exact(&query(20), 200).await.unwrap(); assert_eq!(exact_result.candidates, indexed_result.candidates); + assert_eq!( + exact_result.candidates, + CanonicalCandidates::new(indexed_exact_result).truncated(20) + ); assert_eq!( exact_result.completeness, VectorRecallCompleteness::Exhaustive { scanned: 200 } diff --git a/src/api/types/retrieval.rs b/src/api/types/retrieval.rs index 9d036a4b..09ddab73 100644 --- a/src/api/types/retrieval.rs +++ b/src/api/types/retrieval.rs @@ -280,6 +280,14 @@ pub struct RetrievalTelemetry { /// Completeness of the vector candidate set reported for a retrieval. /// +/// `NotRequested` means no vector search ran. `Exhaustive` means an unindexed +/// shard scanned the full closed scope, and `scanned` is that scope's population. +/// `BoundaryTieClosed` means the shared fetch loop closed the cutoff score cohort +/// within an indexed result prefix; `fetched` is the number of backend rows read. +/// `BoundaryTieOpen` means the cohort was still open at `fetch_bound`; `fetched` +/// is the number of rows read. Indexed boundary verdicts are deterministic only +/// for the index-returned prefix and never claim population-level completeness. +/// /// ``` /// use character_memory::api::types::VectorRecallCompleteness as ApiCompleteness; /// use character_memory::VectorRecallCompleteness; diff --git a/src/memory.rs b/src/memory.rs index d08111bc..c8f22e55 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -175,12 +175,12 @@ mod tests { use crate::policy::memory_object_vector_record; use crate::test_support::{ representative_fixtures, DeterministicMemoryEmbedder, FakeGraphAuthorityStore, - FakeVectorCandidateStore, + TemporaryVectorCandidateStore, }; #[tokio::test] async fn injected_facade_remembers_through_the_write_plan_path() { - let memory = injected_memory(); + let memory = injected_memory().await; let entity_id = id("550e8400-e29b-41d4-a716-446655445001"); let mut entity = EntityDraft::new(EntityType::User, "Kohta"); entity.id = Some(entity_id); @@ -201,7 +201,7 @@ mod tests { #[tokio::test] async fn remember_surfaces_write_plan_validation_warnings() { - let memory = injected_memory(); + let memory = injected_memory().await; let episode_id = id("550e8400-e29b-41d4-a716-446655445011"); let mut episode = EpisodeDraft::new("echoed source content"); episode.id = Some(episode_id); @@ -250,7 +250,7 @@ mod tests { #[tokio::test] async fn prepare_and_validate_plan_do_not_persist() { - let memory = injected_memory(); + let memory = injected_memory().await; let plan = memory .prepare( @@ -283,7 +283,7 @@ mod tests { #[tokio::test] async fn commit_revalidates_against_current_graph_state() { - let memory = injected_memory(); + let memory = injected_memory().await; let missing_entity_id = id("550e8400-e29b-41d4-a716-446655445021"); let plan = memory .prepare( @@ -307,7 +307,7 @@ mod tests { #[tokio::test] async fn remember_returns_structured_validation_rejection() { - let memory = injected_memory(); + let memory = injected_memory().await; let missing_entity_id = id("550e8400-e29b-41d4-a716-446655445022"); let error = memory @@ -327,7 +327,7 @@ mod tests { #[tokio::test] async fn commit_retry_is_idempotent_and_rejects_divergent_content() { - let memory = injected_memory(); + let memory = injected_memory().await; let mut plan = memory .prepare( RememberInput::new("same content"), @@ -406,7 +406,7 @@ mod tests { #[tokio::test] async fn injected_facade_links_canonical_relationships() { - let memory = injected_memory(); + let memory = injected_memory().await; let from_id = id("550e8400-e29b-41d4-a716-446655445010"); let to_id = id("550e8400-e29b-41d4-a716-446655445011"); let mut draft = MemoryLinkDraft::new( @@ -460,7 +460,7 @@ mod tests { #[tokio::test] async fn retrieve_rejects_an_empty_configured_object_type_scope_at_the_boundary() { - let memory = injected_memory(); + let memory = injected_memory().await; let mut context = RetrievalContext::new("invalid empty scope"); context.object_type_defaults.clear(); @@ -854,10 +854,10 @@ mod tests { )); } - fn injected_memory() -> CharacterMemory { + async fn injected_memory() -> CharacterMemory { CharacterMemory::from_parts( Box::new(FakeGraphAuthorityStore::new()), - Box::new(FakeVectorCandidateStore::new()), + Box::new(TemporaryVectorCandidateStore::open(8).await), Box::new(DeterministicMemoryEmbedder::new(8)), ) } @@ -901,7 +901,7 @@ mod tests { let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let objects = [ MemoryObject::Episode(fixtures.episode.clone()), MemoryObject::Observation(fixtures.salient_observation.clone()), diff --git a/src/models/vector.rs b/src/models/vector.rs index 560c152f..b4112aba 100644 --- a/src/models/vector.rs +++ b/src/models/vector.rs @@ -6,8 +6,6 @@ mod record; pub(crate) use crate::domain::VectorSurface; #[cfg(any(test, feature = "test-fixtures"))] use crate::domain::{MemoryId, MemoryObjectRef, ObjectType, DEFAULT_SCHEMA_VERSION}; -#[cfg(test)] -pub(crate) use candidate_record::VectorCandidateRecord; pub(crate) use candidate_record::{ CanonicalCandidates, EmbeddingInput, VectorCandidateMatch, VectorCandidateSearch, }; diff --git a/src/models/vector/candidate_record.rs b/src/models/vector/candidate_record.rs index bf78bebc..05762678 100644 --- a/src/models/vector/candidate_record.rs +++ b/src/models/vector/candidate_record.rs @@ -1,5 +1,4 @@ -// Vector candidate query surface. Some filters are exercised by live -// adapters while deterministic tests use narrower subsets. +// Provider-neutral vector candidate query and match types shared by both adapters. use std::collections::{hash_map::Entry, HashMap}; use crate::domain::{MemoryId, ObjectType, VectorSurface}; @@ -28,32 +27,6 @@ impl EmbeddingInput { } } -#[derive(Debug, Clone, PartialEq)] -#[cfg(test)] -pub(crate) struct VectorCandidateRecord { - pub(crate) object_id: MemoryId, - pub(crate) object_type: ObjectType, - pub(crate) surface: VectorSurface, - pub(crate) embedding: Vec, -} - -#[cfg(test)] -impl VectorCandidateRecord { - pub(crate) fn new( - object_id: MemoryId, - object_type: ObjectType, - surface: VectorSurface, - embedding: Vec, - ) -> Self { - Self { - object_id, - object_type, - surface, - embedding, - } - } -} - #[derive(Debug, Clone, PartialEq)] pub(crate) struct VectorCandidateSearch { pub(crate) query_embedding: Vec, @@ -181,22 +154,6 @@ fn vector_surface_rank(surface: VectorSurface) -> u8 { mod tests { use super::*; - #[test] - fn vector_candidate_record_keeps_domain_identity_and_embedding_surface() { - let object_id = MemoryId::new_v4(); - let record = VectorCandidateRecord::new( - object_id, - ObjectType::Observation, - VectorSurface::Text, - vec![0.1, 0.2, 0.3], - ); - - assert_eq!(record.object_id, object_id); - assert_eq!(record.object_type, ObjectType::Observation); - assert_eq!(record.surface, VectorSurface::Text); - assert_eq!(record.embedding, vec![0.1, 0.2, 0.3]); - } - #[test] fn vector_candidate_search_can_scope_by_canonical_object_types() { let search = VectorCandidateSearch::new( diff --git a/src/models/vector/record.rs b/src/models/vector/record.rs index 638d66c5..82db0b41 100644 --- a/src/models/vector/record.rs +++ b/src/models/vector/record.rs @@ -3,8 +3,6 @@ use crate::domain::{MemoryId, ObjectType, VectorSurface}; use super::EmbeddingInput; -#[cfg(test)] -use super::VectorCandidateRecord; #[derive(Debug, Clone, Copy)] pub(crate) struct VectorRecordEmbedding<'a> { @@ -16,11 +14,6 @@ impl<'a> VectorRecordEmbedding<'a> { pub(crate) fn new(record: &'a VectorRecord, embedding: &'a [f32]) -> Self { Self { record, embedding } } - - #[cfg(test)] - pub(crate) fn to_candidate_record(self) -> VectorCandidateRecord { - self.record.to_candidate_record(self.embedding.to_vec()) - } } #[derive(Debug, Clone, PartialEq)] @@ -57,11 +50,6 @@ impl VectorRecord { self.embedding_text.clone(), ) } - - #[cfg(test)] - pub(crate) fn to_candidate_record(&self, embedding: Vec) -> VectorCandidateRecord { - VectorCandidateRecord::new(self.object_id, self.object_type, self.surface, embedding) - } } impl From<&VectorRecord> for EmbeddingInput { @@ -95,23 +83,4 @@ mod tests { assert!(!input.text.contains(&object_id.to_string())); assert!(!input.text.contains(DEFAULT_SCHEMA_VERSION)); } - - #[test] - fn vector_record_converts_to_existing_candidate_contract_with_embedding() { - let object_id = MemoryId::new_v4(); - let record = VectorRecord::new( - object_id, - ObjectType::Observation, - VectorSurface::Text, - DEFAULT_SCHEMA_VERSION, - "Observation excerpt: Use deterministic fakes.", - ); - - let candidate = record.to_candidate_record(vec![0.1, 0.2]); - - assert_eq!(candidate.object_id, object_id); - assert_eq!(candidate.object_type, ObjectType::Observation); - assert_eq!(candidate.surface, VectorSurface::Text); - assert_eq!(candidate.embedding, vec![0.1, 0.2]); - } } diff --git a/src/policy.rs b/src/policy.rs index ba504e96..4d5c5d12 100644 --- a/src/policy.rs +++ b/src/policy.rs @@ -5,8 +5,6 @@ pub(crate) mod embedding_surface; pub(crate) mod graph_expansion; pub(crate) mod retrieval_selectivity; -#[cfg(test)] -pub(crate) use embedding_surface::episode_vector_record; pub(crate) use embedding_surface::memory_object_vector_record; pub(crate) use retrieval_selectivity::{ selectivity_plan_for_candidate, RetrievalSelectivityPolicy, SelectivityPlan, diff --git a/src/ports/vector_candidate.rs b/src/ports/vector_candidate.rs index 2abd68e4..1dcf672e 100644 --- a/src/ports/vector_candidate.rs +++ b/src/ports/vector_candidate.rs @@ -1,5 +1,4 @@ -// Vector candidate recall contract. Qdrant is the default adapter, while -// tests use deterministic fake stores. +// Vector candidate recall contract shared by the embedded and service adapters. use async_trait::async_trait; use crate::api::types::retrieval::VectorRecallCompleteness; @@ -22,6 +21,11 @@ pub(crate) trait VectorCandidateStore: Send + Sync { /// Returns at most `query.limit` unique object/surface matches in canonical /// score-descending, object-type, object-id, surface order. + /// + /// The shared fetch loop closes every score tie that crosses the requested + /// limit. `Exhaustive` is reported only for a closed cohort from an unindexed + /// shard; indexed recall reports whether the returned prefix closed its + /// boundary tie or remained open at the fetch bound. async fn search_candidates( &self, query: &VectorCandidateSearch, diff --git a/src/test_support.rs b/src/test_support.rs index 0f1b0fc0..42b35e1b 100644 --- a/src/test_support.rs +++ b/src/test_support.rs @@ -1,13 +1,12 @@ // Deterministic test harness shared by pipeline, adapter, and facade tests. -use std::collections::HashSet; use std::sync::{Mutex, MutexGuard}; use async_trait::async_trait; use chrono::{DateTime, Utc}; use uuid::Uuid; -use crate::api::types::retrieval::VectorRecallCompleteness; +use crate::adapters::qdrant_edge::QdrantEdgeVectorCandidateStore; use crate::domain::{ DerivedMemory, DerivedType, Entity, EntityType, Episode, MemoryId, MemoryLink, MemoryObject, MemoryObjectRef, MemoryThread, Modality, ObjectType, Observation, RelationType, RetentionState, @@ -15,8 +14,7 @@ use crate::domain::{ }; use crate::errors::{CustomError, GraphQueryError}; use crate::models::vector::{ - CanonicalCandidates, EmbeddingInput, VectorCandidateMatch, VectorCandidateRecord, - VectorCandidateSearch, VectorRecordEmbedding, VectorSurface, + EmbeddingInput, VectorCandidateSearch, VectorRecordEmbedding, VectorSurface, }; use crate::policy::graph_expansion::{ bounded_expansion, derived_memories_by_provenance, derived_memories_by_thread, @@ -28,115 +26,50 @@ use crate::ports::graph_authority::{ }; use crate::ports::vector_candidate::{VectorCandidateRecall, VectorCandidateStore}; -#[derive(Debug, Default)] -pub(crate) struct FakeVectorCandidateStore { - records: Mutex>, +#[derive(Debug)] +pub(crate) struct TemporaryVectorCandidateStore { + store: QdrantEdgeVectorCandidateStore, + _directory: tempfile::TempDir, } -impl FakeVectorCandidateStore { - pub(crate) fn new() -> Self { - Self::default() - } - - pub(crate) async fn upsert_candidates( - &self, - candidates: &[VectorCandidateRecord], - ) -> Result<(), CustomError> { - self.replace_candidates(candidates) - } - - fn replace_candidates(&self, candidates: &[VectorCandidateRecord]) -> Result<(), CustomError> { - let mut records = lock(&self.records)?; - - for candidate in candidates { - records.retain(|record| { - record.object_id != candidate.object_id || record.surface != candidate.surface - }); - records.push(candidate.clone()); +impl TemporaryVectorCandidateStore { + pub(crate) async fn open(vector_size: usize) -> Self { + let directory = tempfile::TempDir::new().expect("temporary vector directory"); + let store = QdrantEdgeVectorCandidateStore::open( + directory.path(), + format!("test_{}", Uuid::new_v4().simple()), + vector_size, + ) + .await + .expect("temporary embedded vector store"); + Self { + store, + _directory: directory, } - - Ok(()) } } #[async_trait] -impl VectorCandidateStore for FakeVectorCandidateStore { +impl VectorCandidateStore for TemporaryVectorCandidateStore { async fn upsert_vector_records( &self, records: &[VectorRecordEmbedding<'_>], ) -> Result<(), CustomError> { - let candidates = records - .iter() - .map(|record| record.to_candidate_record()) - .collect::>(); - self.replace_candidates(&candidates) + self.store.upsert_vector_records(records).await } async fn search_candidates( &self, query: &VectorCandidateSearch, ) -> Result { - if query.limit == 0 || query.object_types.is_empty() { - return Ok(VectorCandidateRecall { - candidates: CanonicalCandidates::new([]), - completeness: VectorRecallCompleteness::NotRequested, - }); - } - - let records = lock(&self.records)?; - let matches: Vec<_> = records - .iter() - .filter(|record| query.object_types.contains(&record.object_type)) - .map(|record| { - VectorCandidateMatch::new( - record.object_id, - record.object_type, - record.surface, - cosine_similarity(&query.query_embedding, &record.embedding), - ) - }) - .collect(); - - let scanned = matches.len(); - Ok(VectorCandidateRecall { - candidates: CanonicalCandidates::new(matches).truncated(query.limit), - completeness: VectorRecallCompleteness::Exhaustive { scanned }, - }) + self.store.search_candidates(query).await } async fn delete_candidates(&self, object_ids: &[MemoryId]) -> Result<(), CustomError> { - let delete_ids: HashSet<_> = object_ids.iter().copied().collect(); - lock(&self.records)?.retain(|record| !delete_ids.contains(&record.object_id)); - Ok(()) + self.store.delete_candidates(object_ids).await } } -pub(crate) fn zero_norm_vector_fixture() -> (Vec, VectorCandidateSearch) { - ( - vec![ - VectorCandidateRecord::new( - Uuid::from_u128(1), - ObjectType::Episode, - VectorSurface::Summary, - vec![1.0, 0.0], - ), - VectorCandidateRecord::new( - Uuid::from_u128(2), - ObjectType::Episode, - VectorSurface::Summary, - vec![0.0, 1.0], - ), - VectorCandidateRecord::new( - Uuid::from_u128(3), - ObjectType::Observation, - VectorSurface::Text, - vec![1.0, 0.0], - ), - ], - VectorCandidateSearch::new(vec![0.0, 0.0], 10, vec![ObjectType::Episode]), - ) -} - #[derive(Debug, Default)] pub(crate) struct FakeGraphAuthorityStore { objects: Mutex>, @@ -962,26 +895,6 @@ mod lifecycle_tests { } } -fn cosine_similarity(left: &[f32], right: &[f32]) -> f32 { - if left.len() != right.len() { - return 0.0; - } - - let dot_product: f32 = left - .iter() - .zip(right.iter()) - .map(|(left, right)| left * right) - .sum(); - let left_magnitude = left.iter().map(|value| value * value).sum::().sqrt(); - let right_magnitude = right.iter().map(|value| value * value).sum::().sqrt(); - - if left_magnitude == 0.0 || right_magnitude == 0.0 { - 0.0 - } else { - dot_product / (left_magnitude * right_magnitude) - } -} - fn deterministic_embedding(input: &EmbeddingInput, dimensions: usize) -> Vec { let mut embedding = vec![0.0; dimensions]; if dimensions == 0 { @@ -1017,154 +930,6 @@ mod tests { GraphExpansionFilteredReason, GraphExpansionLifecyclePolicy, }; - #[tokio::test] - async fn vector_fake_upserts_searches_and_deletes_deterministically() { - let store = FakeVectorCandidateStore::new(); - let fixtures = representative_fixtures(); - - store - .upsert_candidates(&[ - VectorCandidateRecord::new( - fixtures.episode.id, - ObjectType::Episode, - VectorSurface::Summary, - vec![1.0, 0.0], - ), - VectorCandidateRecord::new( - fixtures.salient_observation.id, - ObjectType::Observation, - VectorSurface::Text, - vec![0.0, 1.0], - ), - ]) - .await - .unwrap(); - - let query = VectorCandidateSearch::new( - vec![1.0, 0.0], - 10, - vec![ObjectType::Episode, ObjectType::Observation], - ); - let first_result = store.search_candidates(&query).await.unwrap(); - let second_result = store.search_candidates(&query).await.unwrap(); - - assert_eq!(first_result, second_result); - assert_eq!( - first_result.completeness, - VectorRecallCompleteness::Exhaustive { scanned: 2 } - ); - assert_eq!(first_result.candidates[0].object_id, fixtures.episode.id); - assert_eq!(first_result.candidates[0].object_type, ObjectType::Episode); - assert_eq!(first_result.candidates[0].surface, VectorSurface::Summary); - - store - .delete_candidates(&[fixtures.episode.id]) - .await - .unwrap(); - let after_delete = store.search_candidates(&query).await.unwrap(); - - assert_eq!(after_delete.candidates.len(), 1); - assert_eq!( - after_delete.candidates[0].object_id, - fixtures.salient_observation.id - ); - } - - #[tokio::test] - async fn vector_fake_canonicalizes_equal_scores_before_limit_across_insertion_orders() { - let first_store = FakeVectorCandidateStore::new(); - let second_store = FakeVectorCandidateStore::new(); - let first_episode_id = Uuid::from_u128(2); - let second_episode_id = Uuid::from_u128(3); - let observation_id = Uuid::from_u128(1); - let candidates = vec![ - VectorCandidateRecord::new( - observation_id, - ObjectType::Observation, - VectorSurface::Text, - vec![1.0, 0.0], - ), - VectorCandidateRecord::new( - second_episode_id, - ObjectType::Episode, - VectorSurface::Summary, - vec![1.0, 0.0], - ), - VectorCandidateRecord::new( - first_episode_id, - ObjectType::Episode, - VectorSurface::Summary, - vec![1.0, 0.0], - ), - ]; - let mut reversed = candidates.clone(); - reversed.reverse(); - - first_store.upsert_candidates(&candidates).await.unwrap(); - second_store.upsert_candidates(&reversed).await.unwrap(); - - let query = VectorCandidateSearch::new( - vec![1.0, 0.0], - 2, - vec![ObjectType::Episode, ObjectType::Observation], - ); - let first = first_store.search_candidates(&query).await.unwrap(); - let second = second_store.search_candidates(&query).await.unwrap(); - - assert_eq!(first, second); - assert_eq!( - first - .candidates - .iter() - .map(|candidate| candidate.object_id) - .collect::>(), - vec![first_episode_id, second_episode_id] - ); - } - - #[tokio::test] - async fn vector_fake_upserts_full_records_through_store_contract() { - let store = FakeVectorCandidateStore::new(); - let fixtures = representative_fixtures(); - let record = crate::policy::episode_vector_record(&fixtures.episode); - let records = vec![VectorRecordEmbedding::new(&record, &[1.0, 0.0])]; - - store.upsert_vector_records(&records).await.unwrap(); - - let matches = store - .search_candidates(&VectorCandidateSearch::new( - vec![1.0, 0.0], - 10, - vec![ObjectType::Episode], - )) - .await - .unwrap(); - - assert_eq!(matches.candidates.len(), 1); - assert_eq!(matches.candidates[0].object_id, fixtures.episode.id); - assert_eq!(matches.candidates[0].object_type, ObjectType::Episode); - assert_eq!(matches.candidates[0].surface, VectorSurface::Summary); - } - - #[tokio::test] - async fn vector_fake_zero_norm_fixture_scores_every_scoped_candidate_zero() { - let store = FakeVectorCandidateStore::new(); - let (records, query) = zero_norm_vector_fixture(); - store.upsert_candidates(&records).await.unwrap(); - - let recall = store.search_candidates(&query).await.unwrap(); - - assert_eq!(recall.candidates.len(), 2); - assert!(recall - .candidates - .iter() - .all(|candidate| candidate.score == 0.0)); - assert_eq!( - recall.completeness, - VectorRecallCompleteness::Exhaustive { scanned: 2 } - ); - } - #[tokio::test] async fn graph_fake_preserves_objects_links_lifecycle_and_raw_refs() { let store = FakeGraphAuthorityStore::new(); @@ -1513,9 +1278,4 @@ mod tests { .iter() .any(|link| link.from_id == fixtures.hub_entity.id)); } - - #[test] - fn cosine_similarity_rejects_mismatched_dimensions() { - assert_eq!(cosine_similarity(&[1.0, 0.0], &[1.0, 0.0, 0.0]), 0.0); - } } diff --git a/src/usecases/correct_forget.rs b/src/usecases/correct_forget.rs index 6f6d82a8..a9e5f4c9 100644 --- a/src/usecases/correct_forget.rs +++ b/src/usecases/correct_forget.rs @@ -1303,7 +1303,7 @@ mod tests { use crate::ports::vector_candidate::VectorCandidateRecall; use crate::test_support::{ representative_fixtures, DeterministicMemoryEmbedder, FakeGraphAuthorityStore, - FakeVectorCandidateStore, + TemporaryVectorCandidateStore, }; use crate::usecases::RetrievePipeline; @@ -1416,7 +1416,7 @@ mod tests { ]) .await .unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(8).await; let embedder = DeterministicMemoryEmbedder::new(8); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); @@ -1461,7 +1461,7 @@ mod tests { ]) .await .unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(8).await; let embedder = DeterministicMemoryEmbedder::new(8); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let draft = stateful_correction_draft(&ids, "Stable corrected payload."); @@ -1564,7 +1564,7 @@ mod tests { let ids = fixed_ids(); let old = old_memory(&ids); let graph = RecordingGraphStore::new(vec![MemoryObject::DerivedMemory(old.clone())]); - let vector = OneShotDeleteFailingVectorStore::new(); + let vector = OneShotDeleteFailingVectorStore::new().await; let old_record = memory_object_vector_record(&MemoryObject::DerivedMemory(old)).unwrap(); vector .inner @@ -1700,7 +1700,7 @@ mod tests { ]) .await .unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(8).await; let embedder = DeterministicMemoryEmbedder::new(8); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let first_ancestor = MemoryId::from_u128(0x550e_8400_e29b_41d4_a716_4466_5544_8201); @@ -2330,7 +2330,7 @@ mod tests { let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let replacement_id = Uuid::from_u128(0x550e_8400_e29b_41d4_a716_4466_5544_9100); @@ -2426,7 +2426,7 @@ mod tests { objects.push(MemoryObject::DerivedMemory(observation_only.clone())); graph.upsert_objects(&objects).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let replacement_id = Uuid::from_u128(0x550e_8400_e29b_41d4_a716_4466_5544_9102); @@ -2593,7 +2593,7 @@ mod tests { "Establish current replacement.", )); graph.upsert_links(&links).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let mut replacement = ReplacementDerivedMemoryDraft::new( @@ -2658,7 +2658,7 @@ mod tests { "Establish current replacement.", )); graph.upsert_links(&links).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); @@ -2703,7 +2703,7 @@ mod tests { let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let mut draft = ForgetMemoryDraft::suppress( @@ -2735,7 +2735,7 @@ mod tests { } graph.upsert_objects(&objects).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); @@ -2800,7 +2800,7 @@ mod tests { objects.push(MemoryObject::DerivedMemory(observation_only.clone())); graph.upsert_objects(&objects).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); @@ -2837,7 +2837,7 @@ mod tests { let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let mut draft = ForgetMemoryDraft::suppress( @@ -2889,7 +2889,7 @@ mod tests { let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let mut draft = ForgetMemoryDraft::suppress( @@ -2924,7 +2924,7 @@ mod tests { let fixtures = representative_fixtures(); let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); @@ -2960,7 +2960,7 @@ mod tests { let fixtures = representative_fixtures(); let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(4).await; let embedder = DeterministicMemoryEmbedder::new(4); let pipeline = CorrectionForgetPipeline::new(&graph, &vector, &embedder); let mut draft = @@ -3038,7 +3038,7 @@ mod tests { .upsert_objects(&[MemoryObject::DerivedMemory(old_memory(&ids))]) .await .unwrap(); - let vector = DeleteFailingVectorStore::new(); + let vector = DeleteFailingVectorStore::new().await; vector .inner .upsert_vector_records(&[VectorRecordEmbedding::new( @@ -3087,7 +3087,7 @@ mod tests { let graph = FakeGraphAuthorityStore::new(); graph.upsert_objects(&fixtures.objects()).await.unwrap(); graph.upsert_links(&fixtures.links()).await.unwrap(); - let vector = DeleteFailingVectorStore::new(); + let vector = DeleteFailingVectorStore::new().await; let embedder = DeterministicMemoryEmbedder::new(4); for object in [ MemoryObject::Episode(fixtures.episode.clone()), @@ -3643,14 +3643,14 @@ mod tests { #[derive(Debug)] struct OneShotDeleteFailingVectorStore { - inner: FakeVectorCandidateStore, + inner: TemporaryVectorCandidateStore, fail_next_delete: Mutex, } impl OneShotDeleteFailingVectorStore { - fn new() -> Self { + async fn new() -> Self { Self { - inner: FakeVectorCandidateStore::new(), + inner: TemporaryVectorCandidateStore::open(4).await, fail_next_delete: Mutex::new(true), } } @@ -3750,13 +3750,13 @@ mod tests { #[derive(Debug)] struct DeleteFailingVectorStore { - inner: FakeVectorCandidateStore, + inner: TemporaryVectorCandidateStore, } impl DeleteFailingVectorStore { - fn new() -> Self { + async fn new() -> Self { Self { - inner: FakeVectorCandidateStore::new(), + inner: TemporaryVectorCandidateStore::open(4).await, } } } diff --git a/src/usecases/retrieve.rs b/src/usecases/retrieve.rs index e61ca08f..36d5e06d 100644 --- a/src/usecases/retrieve.rs +++ b/src/usecases/retrieve.rs @@ -1352,12 +1352,12 @@ mod tests { ContinuitySectionLimits, RetrievalCandidateLimits, RetrievalLifecyclePolicy, }; use crate::domain::RetentionState; - use crate::models::vector::{CanonicalCandidates, VectorCandidateRecord}; + use crate::models::vector::{CanonicalCandidates, VectorRecordEmbedding}; use crate::policy::RetrievalSelectivityPolicy; use crate::ports::retrieval_stats::RetrievalStatsEdge; use crate::test_support::{ high_fanout_graph_fixture, representative_fixtures, FakeGraphAuthorityStore, - FakeVectorCandidateStore, + TemporaryVectorCandidateStore, }; #[tokio::test] @@ -2574,14 +2574,13 @@ mod tests { async fn graph_authority_filters_lifecycle_state_after_vector_recall() { let fixtures = representative_fixtures(); let graph = graph_with(&fixtures.objects(), &fixtures.links()).await; - let vector = FakeVectorCandidateStore::new(); + let vector = TemporaryVectorCandidateStore::open(2).await; + let record = crate::policy::memory_object_vector_record(&MemoryObject::DerivedMemory( + fixtures.user_preference.clone(), + )) + .unwrap(); vector - .upsert_candidates(&[VectorCandidateRecord::new( - fixtures.user_preference.id, - ObjectType::DerivedMemory, - VectorSurface::DerivedText, - vec![1.0, 0.0], - )]) + .upsert_vector_records(&[VectorRecordEmbedding::new(&record, &[1.0, 0.0])]) .await .unwrap(); let embedder = RecordingEmbedder::new(vec![1.0, 0.0]); From f91534fd4b035dc6ad50c7812b49a51529fc8078 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 08:05:20 +0900 Subject: [PATCH 02/14] docs: clarify vector store constructor settings --- .../v0-1-6-embedded-vector-recall-plan.md | 3 ++- src/composition.rs | 22 ++++++++++++------- src/config/app_settings.rs | 11 ++++++---- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 371bcc37..918051cf 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -177,6 +177,7 @@ - src/**/tests (fake stores only) - src/api/types/retrieval.rs (doc comments only) - src/ports/vector_candidate.rs (doc comments only) + - src/composition.rs, src/config/app_settings.rs (public constructor doc comments only; deferred from the Task_4 Copilot review) - src/adapters/qdrant/store.rs, src/adapters/qdrant/payload.rs, src/adapters/qdrant_edge/mod.rs (Task_8 audit dispositions only) - src/models/vector/candidate_record.rs (header comment only) - tests/vector_port_contract_tests.rs, tests/support/** (point-id parity fixture and fake-retirement fallout) @@ -193,7 +194,7 @@ - acceptance: - Zero-hit census for the retired fake and record type. - All five checklist rows cite evidence in the Progress Log. - - Task_8 audit dispositions landed: one point-identity derivation (the v5 derivation) shared by both adapters with a parity assertion and no compat shim; one shared read function for the record contract returning the closed error vocabulary, with the service adapter's stringly database errors for payload and scroll-limit faults removed; the canary pins that an indexed shard searched exactly returns the exhaustive result; the service zero-norm verdict's `scanned` comes from the scope count or the loop invariant is stated; the duplicate vector-config validation is reduced to one or justified in place; the stale header comments and README line are corrected. + - Task_8 audit dispositions landed: one point-identity derivation (the v5 derivation) shared by both adapters with a parity assertion and no compat shim; one shared read function for the record contract returning the closed error vocabulary, with the service adapter's stringly database errors for payload and scroll-limit faults removed; the canary pins that an indexed shard searched exactly returns the exhaustive result; the service zero-norm verdict's `scanned` comes from the scope count or the loop invariant is stated; the duplicate vector-config validation is reduced to one or justified in place; the stale header comments and README line are corrected; the public facade constructor and `Settings::new` rustdoc describe both vector-store modes in backend-neutral terms (embedded default with a store path; explicit service mode with a connection string). - The public completeness type documents all four verdict meanings and the `scanned` and `fetched` counters, stating that a closed boundary verdict is deterministic only for the index-returned prefix, never population-level completeness; the port trait's doc comment states the verdict guarantees (tie closure through the shared loop; exhaustive only on an unindexed shard with a closed cohort; open at the bound) so a future adapter cannot label an indexed prefix exhaustive (deferred from the Task_2 review to avoid re-cascading the stack mid-wave). - validation: - kind: command diff --git a/src/composition.rs b/src/composition.rs index 9bdd6618..7f008ff2 100644 --- a/src/composition.rs +++ b/src/composition.rs @@ -100,14 +100,16 @@ impl CharacterMemory { /// # Description /// /// This constructor allows callers to inject custom embedding generation while using the - /// default graph-authoritative storage composition. + /// default graph-authoritative storage composition. Vector candidate recall uses the embedded + /// store by default and requires `VECTOR_STORE_PATH`; callers can explicitly select service + /// mode, which instead requires `QDRANT_CONNECTION_STRING`. /// /// # Parameters /// - /// - `settings`: Global configuration used to derive the Qdrant connection and embedding - /// model settings required to initialize the Qdrant candidate collection. - /// - `collection_name`: The name of the Qdrant collection where memory vectors will be - /// stored and queried. + /// - `settings`: Global configuration used to select and initialize the vector candidate + /// backend and embedding model. + /// - `collection_name`: The name of the vector collection where memory vectors will be stored + /// and queried. /// - `embed_provider`: A boxed implementation of [`EmbeddingProvider`] that is responsible /// for generating embeddings from input data. /// @@ -115,10 +117,10 @@ impl CharacterMemory { /// /// A `Result` which is: /// - /// - `Ok(Self)`: A new [`CharacterMemory`] instance backed by Oxigraph graph authority and - /// Qdrant vector candidate recall. + /// - `Ok(Self)`: A new [`CharacterMemory`] instance backed by Oxigraph graph authority and the + /// configured vector candidate store. /// - `Err(CustomError)`: Returned if any error occurs while resolving configuration from - /// `settings` or initializing the Oxigraph graph authority and Qdrant vector candidate + /// `settings` or initializing the Oxigraph graph authority and configured vector candidate /// store. pub async fn new_with_embedding_provider( settings: Settings, @@ -185,6 +187,10 @@ impl CharacterMemory { /// Constructs a new CharacterMemory instance. /// + /// Vector candidate recall uses the embedded store by default and requires + /// `VECTOR_STORE_PATH`. Explicit service mode instead requires + /// `QDRANT_CONNECTION_STRING`. + /// /// # Parameters /// /// - `settings`: Configuration settings for the memory system diff --git a/src/config/app_settings.rs b/src/config/app_settings.rs index 13af737c..6f7b1a10 100644 --- a/src/config/app_settings.rs +++ b/src/config/app_settings.rs @@ -223,13 +223,16 @@ impl Settings { /// /// # Description /// - /// Primary constructor for creating a Settings instance. Takes a pre-configured Config object that defines all required settings. - /// This allows for flexible configuration sourcing while maintaining a clean initialization interface. + /// Primary constructor for creating a Settings instance. Takes a pre-configured Config object + /// that defines all required settings. This allows for flexible configuration sourcing while + /// maintaining a clean initialization interface. /// /// # Parameters /// - /// - `config`: A `config::Config` instance containing all required settings: - /// - `qdrant_connection_string`: Connection string for Qdrant database + /// - `config`: A `config::Config` instance containing settings required by the selected modes: + /// - `vector_store_mode`: Optional vector mode selector; defaults to embedded + /// - `vector_store_path`: Local directory required by the default embedded vector mode + /// - `qdrant_connection_string`: Connection string required only in explicit service mode /// - `oxigraph_path`: Local filesystem path for the Oxigraph database /// - `openai_api_key`: API key for OpenAI services /// From 8ffb1fc56a7d9638819c9289d984a618765bf476 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 08:49:57 +0900 Subject: [PATCH 03/14] fix: address vector closeout review findings --- docs/coding-agent/lessons.md | 92 +++++++++++++++++++ .../v0-1-6-embedded-vector-recall-plan.md | 5 +- ...ness-and-prefilters-never-match-unknown.md | 4 +- src/adapters/qdrant/payload.rs | 2 - src/adapters/qdrant/store.rs | 24 +++-- src/adapters/qdrant_edge/mod.rs | 2 +- src/api/types/retrieval.rs | 8 +- src/ports/vector_candidate.rs | 8 +- src/test_support.rs | 44 +++++++-- 9 files changed, 161 insertions(+), 28 deletions(-) diff --git a/docs/coding-agent/lessons.md b/docs/coding-agent/lessons.md index 30766499..167f47cc 100644 --- a/docs/coding-agent/lessons.md +++ b/docs/coding-agent/lessons.md @@ -589,3 +589,95 @@ Prevention: Evidence: - Corrected ignored run: 4 passed, 0 failed, 396 filtered out; the exclusive Qdrant window was then released. + +## 2026-09-04 — Classify Boundary Conversions By Operation, Not Nearby Helper [tags: review, errors, conversion, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 review revision +- Roles involved: Worker | Reviewer + +Symptom: +- The service adapter classified a `usize`-to-`u32` scroll-limit overflow as `PayloadDeserialization`. + +Root cause: +- The closeout reused the nearby payload-error helper while replacing a stringly error, without checking the semantic kind already used for scope-count width conversion. + +Fix applied: +- Construct the existing typed `Conversion` error directly and update the boundary test to require that kind. + +Prevention: +- When replacing a stringly error, classify the failed operation first and compare sibling conversions before selecting an existing helper. +- Residual risk / waiver: none. + +Evidence: +- Reviewer finding on Task_7 at `src/adapters/qdrant/store.rs`; the corrected overflow assertion requires `VectorDatabaseErrorKind::Conversion`. + +## 2026-09-04 — Give Every Deletion Deliverable Its Own Exact Census [tags: review, deletion, tests, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 review revision +- Roles involved: Worker | Reviewer + +Symptom: +- The fake/type/helper census passed, but the separately required zero test-only payload-field constants still had a `SURFACE_FIELD` alias. + +Root cause: +- The closeout census covered named fake artifacts but did not translate every independent deletion requirement into an exact symbol search. + +Fix applied: +- Delete `SURFACE_FIELD` and have tests use `QdrantPayloadField::Surface.name()` directly. + +Prevention: +- List each deletion deliverable separately and record an exact zero-hit census for each; do not treat one representative census as covering adjacent deletions. +- Residual risk / waiver: none. + +Evidence: +- Reviewer finding on Task_7 and `rg -n "SURFACE_FIELD" src tests` after the correction. + +## 2026-09-04 — Define Completeness By Proven Work, Not Index State [tags: review, docs, vector-recall, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 review revision +- Roles involved: Worker | Reviewer | Orchestrator + +Symptom: +- Public and decision-record wording said `Exhaustive` was available only on an unindexed shard, although the service zero-norm full-scope scroll correctly reports it too. + +Root cause: +- Documentation generalized one implementation path into the verdict's semantic condition instead of checking every branch that produces the public enum. + +Fix applied: +- State that exhaustive means every requested-scope record was scored through a known-exhaustive path, the cutoff cohort closed, and `scanned` came from the scope count; name unindexed scans and full-scope scrolls as examples. + +Prevention: +- Define public verdicts from observable proof conditions and audit every constructor branch before documenting implementation examples. +- Residual risk / waiver: none. + +Evidence: +- Orchestrator ruling during Task_7 review and the service zero-norm parity test that reports `Exhaustive` from a full-scope scroll. + +## 2026-09-04 — Shut Down Background Owners Before Temporary Directories Drop [tags: review, cleanup, windows, tests, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 review revision +- Roles involved: Worker | Reviewer + +Symptom: +- The embedded test fixture dropped its temporary directory immediately after the store destructor only signalled the shard-owner thread, so Windows could attempt deletion while shard files remained open and `TempDir` would ignore the cleanup error. + +Root cause: +- The fixture relied on field drop order but did not distinguish a shutdown signal from an acknowledged owner shutdown. + +Fix applied: +- The fixture destructor moves the store to a helper thread, awaits the adapter's test-only acknowledged close there, joins the helper, and only then lets the temporary directory drop. + +Prevention: +- A temporary fixture that owns a background resource must wait for that resource's shutdown acknowledgement before filesystem cleanup, and a test must assert the directory is actually removed. +- Residual risk / waiver: none. + +Evidence: +- Reviewer finding on Task_7 and `temporary_vector_store_removes_its_directory_on_drop`. diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 918051cf..ab817724 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -195,7 +195,7 @@ - Zero-hit census for the retired fake and record type. - All five checklist rows cite evidence in the Progress Log. - Task_8 audit dispositions landed: one point-identity derivation (the v5 derivation) shared by both adapters with a parity assertion and no compat shim; one shared read function for the record contract returning the closed error vocabulary, with the service adapter's stringly database errors for payload and scroll-limit faults removed; the canary pins that an indexed shard searched exactly returns the exhaustive result; the service zero-norm verdict's `scanned` comes from the scope count or the loop invariant is stated; the duplicate vector-config validation is reduced to one or justified in place; the stale header comments and README line are corrected; the public facade constructor and `Settings::new` rustdoc describe both vector-store modes in backend-neutral terms (embedded default with a store path; explicit service mode with a connection string). - - The public completeness type documents all four verdict meanings and the `scanned` and `fetched` counters, stating that a closed boundary verdict is deterministic only for the index-returned prefix, never population-level completeness; the port trait's doc comment states the verdict guarantees (tie closure through the shared loop; exhaustive only on an unindexed shard with a closed cohort; open at the bound) so a future adapter cannot label an indexed prefix exhaustive (deferred from the Task_2 review to avoid re-cascading the stack mid-wave). + - The public completeness type documents all four verdict meanings and the `scanned` and `fetched` counters, stating that a closed boundary verdict is deterministic only for the index-returned prefix, never population-level completeness; the port trait's doc comment states the verdict guarantees (tie closure through the shared loop; exhaustive only when every scoped record was scored through a path the adapter knows to be exhaustive, including an unindexed scan or a full-scope scroll, with a closed cohort and `scanned` from the scope count; open at the bound) so a future adapter cannot label an index-produced prefix exhaustive (deferred from the Task_2 review to avoid re-cascading the stack mid-wave). - validation: - kind: command required: true @@ -260,7 +260,8 @@ Append-only editing rule (applies to both logs below): when appending an entry, 3. Search completeness: `backend_fetch_cap_reports_an_open_boundary_without_allocating_rows`, `retrieval_telemetry_preserves_every_vector_recall_completeness_verdict`, the live service boundary tests, and `service_and_embedded_admit_identical_candidates_in_identical_order` cover open, telemetry, closed, and cross-adapter exhaustive-versus-closed behavior through the shared tie loop. 4. Hint filter semantics: `rg -n "VectorCandidateFilter|CandidateFilter|match_or_unknown|matches_or_unknown|MatchUnknown" src tests` returned zero hits; the query carries only object-type scope, with its empty-scope behavior and predicate rules pinned by ADR-I-0024 and the port contract tests. 5. Evaluation baseline capability: this repository exposes singleton-scoped traced candidates, completeness telemetry, and `max_embedding_surfaces`; `rg -n "SearchPointsBuilder|QDRANT_OBJECT_ID_FIELD|QDRANT_OBJECT_TYPE_FIELD|QDRANT_CONTENT_TEXT_FIELD" crates/cmem-eval-adapter-cmem/src/lib.rs` still finds the companion repository's direct Qdrant baseline, so its trace migration, row-level identity/rank A/B diff, and post-switch zero-hit census are explicitly pending in that repository. -- 2026-09-04 Task_7 validation closeout: `cargo fmt --all -- --check` passed; `cargo clippy --all-targets --all-features -- -D warnings` passed on Rust 1.97; bare `cargo test` passed 395 library tests, 31 integration tests, and one doc test with five intentional ignores after the point-identity fixture was added; the live-switch `cargo test` passed the same suite with both service/embedded parity tests executing; and the four ignored service-Qdrant tests passed explicitly under the live endpoint. The package is 0.1.6, the phase and roadmap are finished, and this plan is moved to completed. Independent reviewer approval and stack merge remain Orchestrator gates. +- 2026-09-04 Task_7 validation closeout: `cargo fmt --all -- --check` passed; `cargo clippy --all-targets --all-features -- -D warnings` passed on Rust 1.97; bare `cargo test` passed 396 library tests, 31 integration tests, and one doc test with five intentional ignores after the point-identity and temporary-directory-cleanup fixtures were added; the live-switch `cargo test` passed with both service/embedded parity tests executing; and the four ignored service-Qdrant tests passed explicitly under the live endpoint. The package is 0.1.6, the phase and roadmap are finished, and this plan is moved to completed. Independent reviewer approval and stack merge remain Orchestrator gates. +- 2026-09-04 Task_7 review revision: the service scroll-width overflow now returns the typed `Conversion` kind; `rg -n "SURFACE_FIELD" src tests` returns zero hits; completeness docs and ADR-I-0024 define exhaustive recall by proving every scoped record was scored through a known-exhaustive path; and the temporary embedded fixture waits for acknowledged owner shutdown before directory cleanup, with a Windows-sensitive deletion assertion. The service-free suite, strict clippy, strict rustdoc generation, and formatting gate all pass; no live rerun was needed because the delta changes local conversion classification, documentation, and the embedded test fixture only. ## Decision Log (append-only; re-plans and major discoveries) diff --git a/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md b/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md index e7e4df52..2752e5aa 100644 --- a/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md +++ b/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md @@ -39,7 +39,7 @@ A second adapter (ADR-I-0023) makes both gaps matter: below its indexing thresho The search result carries, beside the canonical candidates, a completeness verdict stated by the adapter. The verdict distinguishes four situations: no search was issued because the limit was zero or the scope was empty; every stored record in scope was scored, so the requested top-K is determinate over the population; an index answered with a prefix whose cutoff cohort was closed, so the returned set is determinate for that index state although an approximate index may have omitted records it never surfaced; and the overfetch bound was reached with the cutoff cohort still open, so membership may vary. -Adapters state the verdict truthfully: exhaustive only when the adapter knows the shard is unindexed and the cutoff cohort was closed, with the scanned count taken from the scope rather than the rows returned; an exhaustive scan whose cohort stays open at the bound reports open. +Adapters state the verdict truthfully: exhaustive only when every record in the requested scope was scored through a path the adapter knows to be exhaustive (an unindexed scan or a full-scope scroll such as the zero-norm path) and the cutoff cohort was closed, with the scanned count taken from the scope rather than the rows returned; an exhaustive path whose cohort stays open at the bound reports open. The retrieval pipeline records the verdict in retrieval telemetry beside the returned candidate count and never repairs, retries, or fails on it. Degenerate vectors are defined on both sides of the port so that neither adapter has an undefined path: a zero-norm record embedding is rejected at indexing as a typed per-record failure before any adapter sees it, and a zero-norm query scores every candidate zero and reports a truthful verdict. @@ -93,7 +93,7 @@ Not covered: the current query shape (an embedding, a limit, and an object-type - Unit tests on the service adapter's fetch decision assert the closed and open verdicts, including the all-tied cohort at the bound. - A retrieval test asserts the telemetry verdict for each situation using the fakes. -- The parity suite asserts exhaustive for the embedded adapter below its indexing threshold and closed for the service adapter on the identical-vector tie fixture. +- The parity suite asserts exhaustive for the embedded adapter below its indexing threshold and closed for the service adapter on the identical-vector tie fixture; the zero-norm parity fixture asserts exhaustive for both adapters because each scores the full requested scope. - A census of the vector adapters shows no match-or-unknown condition. - Parity fixtures cover the zero-norm query and the rejected zero-norm record in both adapters. diff --git a/src/adapters/qdrant/payload.rs b/src/adapters/qdrant/payload.rs index 3e3d9819..b4a26279 100644 --- a/src/adapters/qdrant/payload.rs +++ b/src/adapters/qdrant/payload.rs @@ -107,8 +107,6 @@ const fn schema( pub(crate) const OBJECT_ID_FIELD: &str = QdrantPayloadField::ObjectId.name(); pub(crate) const OBJECT_TYPE_FIELD: &str = QdrantPayloadField::ObjectType.name(); -#[cfg(test)] -pub(crate) const SURFACE_FIELD: &str = QdrantPayloadField::Surface.name(); pub(crate) fn qdrant_point_id(record: &VectorRecord) -> MemoryId { MemoryId::new_v5(&record.object_id, record.surface.to_string().as_bytes()) diff --git a/src/adapters/qdrant/store.rs b/src/adapters/qdrant/store.rs index dbd38f18..433221fb 100644 --- a/src/adapters/qdrant/store.rs +++ b/src/adapters/qdrant/store.rs @@ -21,8 +21,8 @@ use crate::models::vector::{VectorCandidateMatch, VectorCandidateSearch, VectorR use crate::ports::vector_candidate::{VectorCandidateRecall, VectorCandidateStore}; use super::payload::{ - payload_deserialization_error, qdrant_payload_map, qdrant_point_id, read_candidate_match, - QdrantPayloadSchema, OBJECT_ID_FIELD, OBJECT_TYPE_FIELD, + qdrant_payload_map, qdrant_point_id, read_candidate_match, QdrantPayloadSchema, + OBJECT_ID_FIELD, OBJECT_TYPE_FIELD, }; use super::tie_closure::close_tie_cohort; @@ -519,8 +519,10 @@ fn qdrant_candidate_config(url: &str) -> QdrantConfig { fn qdrant_scroll_fetch_limit(fetch_limit: usize) -> Result { u32::try_from(fetch_limit).map_err(|_| { - CustomError::VectorDatabaseError(payload_deserialization_error( + CustomError::VectorDatabaseError(VectorDatabaseError::new( "qdrant", + VectorDatabaseErrorKind::Conversion, + None, format!( "Qdrant scroll limit {fetch_limit} exceeds the backend maximum {}", u32::MAX @@ -591,7 +593,7 @@ fn qdrant_payload_to_match( #[cfg(test)] mod tests { use super::*; - use crate::adapters::qdrant::payload::SURFACE_FIELD; + use crate::adapters::qdrant::payload::QdrantPayloadField; use crate::api::types::retrieval::VectorRecallCompleteness; use crate::domain::{ObjectType, VectorSurface, DEFAULT_SCHEMA_VERSION}; use crate::models::vector::{CanonicalCandidates, VectorRecord, VectorRecordEmbedding}; @@ -686,7 +688,7 @@ mod tests { assert!(matches!( qdrant_scroll_fetch_limit(too_large), Err(CustomError::VectorDatabaseError(VectorDatabaseError { - kind: VectorDatabaseErrorKind::PayloadDeserialization, + kind: VectorDatabaseErrorKind::Conversion, .. })) )); @@ -981,7 +983,10 @@ mod tests { string_value(&object_id.to_string()), ), (OBJECT_TYPE_FIELD.to_owned(), string_value("derived_memory")), - (SURFACE_FIELD.to_owned(), string_value("derived_text")), + ( + QdrantPayloadField::Surface.name().to_owned(), + string_value("derived_text"), + ), ]), score: 0.75, ..Default::default() @@ -1064,7 +1069,7 @@ mod tests { "derived_memory" ); assert_eq!( - payload_string(&points[0].payload, SURFACE_FIELD).unwrap(), + payload_string(&points[0].payload, QdrantPayloadField::Surface.name()).unwrap(), "derived_text" ); assert_eq!(points[0].payload.len(), 5); @@ -1292,7 +1297,10 @@ mod tests { OBJECT_TYPE_FIELD.to_owned(), string_value(&object_type.to_string()), ), - (SURFACE_FIELD.to_owned(), string_value("derived_text")), + ( + QdrantPayloadField::Surface.name().to_owned(), + string_value("derived_text"), + ), ]), score, ..Default::default() diff --git a/src/adapters/qdrant_edge/mod.rs b/src/adapters/qdrant_edge/mod.rs index 160566a6..aab1b616 100644 --- a/src/adapters/qdrant_edge/mod.rs +++ b/src/adapters/qdrant_edge/mod.rs @@ -142,7 +142,7 @@ impl QdrantEdgeVectorCandidateStore { } #[cfg(test)] - async fn close(&self) -> Result<(), CustomError> { + pub(crate) async fn close(&self) -> Result<(), CustomError> { let _operation = self.operation.lock().await; let (reply, receiver) = oneshot::channel(); self.send(Command::Shutdown { reply: Some(reply) })?; diff --git a/src/api/types/retrieval.rs b/src/api/types/retrieval.rs index 09ddab73..109c12f4 100644 --- a/src/api/types/retrieval.rs +++ b/src/api/types/retrieval.rs @@ -280,10 +280,12 @@ pub struct RetrievalTelemetry { /// Completeness of the vector candidate set reported for a retrieval. /// -/// `NotRequested` means no vector search ran. `Exhaustive` means an unindexed -/// shard scanned the full closed scope, and `scanned` is that scope's population. +/// `NotRequested` means no vector search ran. `Exhaustive` means every record in +/// the requested scope was scored through a path the adapter knows to be exhaustive, +/// the cutoff cohort closed, and `scanned` is that scope's population. This includes +/// an unindexed scan or a full-scope scroll such as the zero-norm path. /// `BoundaryTieClosed` means the shared fetch loop closed the cutoff score cohort -/// within an indexed result prefix; `fetched` is the number of backend rows read. +/// within an index-produced result prefix; `fetched` is the number of backend rows read. /// `BoundaryTieOpen` means the cohort was still open at `fetch_bound`; `fetched` /// is the number of rows read. Indexed boundary verdicts are deterministic only /// for the index-returned prefix and never claim population-level completeness. diff --git a/src/ports/vector_candidate.rs b/src/ports/vector_candidate.rs index 1dcf672e..5ef20cd0 100644 --- a/src/ports/vector_candidate.rs +++ b/src/ports/vector_candidate.rs @@ -23,9 +23,11 @@ pub(crate) trait VectorCandidateStore: Send + Sync { /// score-descending, object-type, object-id, surface order. /// /// The shared fetch loop closes every score tie that crosses the requested - /// limit. `Exhaustive` is reported only for a closed cohort from an unindexed - /// shard; indexed recall reports whether the returned prefix closed its - /// boundary tie or remained open at the fetch bound. + /// limit. `Exhaustive` is reported only when every record in the requested + /// scope was scored through a path the adapter knows to be exhaustive and the + /// cutoff cohort closed; `scanned` is the scope count. This includes an unindexed + /// scan or a full-scope scroll. An index-produced result prefix reports whether + /// its boundary tie closed or remained open at the fetch bound. async fn search_candidates( &self, query: &VectorCandidateSearch, diff --git a/src/test_support.rs b/src/test_support.rs index 42b35e1b..562d8f7e 100644 --- a/src/test_support.rs +++ b/src/test_support.rs @@ -28,8 +28,8 @@ use crate::ports::vector_candidate::{VectorCandidateRecall, VectorCandidateStore #[derive(Debug)] pub(crate) struct TemporaryVectorCandidateStore { - store: QdrantEdgeVectorCandidateStore, - _directory: tempfile::TempDir, + store: Option, + directory: tempfile::TempDir, } impl TemporaryVectorCandidateStore { @@ -43,10 +43,29 @@ impl TemporaryVectorCandidateStore { .await .expect("temporary embedded vector store"); Self { - store, - _directory: directory, + store: Some(store), + directory, } } + + fn store(&self) -> &QdrantEdgeVectorCandidateStore { + self.store.as_ref().expect("temporary vector store is open") + } +} + +impl Drop for TemporaryVectorCandidateStore { + fn drop(&mut self) { + let store = self.store.take().expect("temporary vector store is open"); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .build() + .expect("temporary vector shutdown runtime") + .block_on(store.close()) + .expect("temporary vector store shutdown"); + }) + .join() + .expect("temporary vector shutdown thread"); + } } #[async_trait] @@ -55,18 +74,18 @@ impl VectorCandidateStore for TemporaryVectorCandidateStore { &self, records: &[VectorRecordEmbedding<'_>], ) -> Result<(), CustomError> { - self.store.upsert_vector_records(records).await + self.store().upsert_vector_records(records).await } async fn search_candidates( &self, query: &VectorCandidateSearch, ) -> Result { - self.store.search_candidates(query).await + self.store().search_candidates(query).await } async fn delete_candidates(&self, object_ids: &[MemoryId]) -> Result<(), CustomError> { - self.store.delete_candidates(object_ids).await + self.store().delete_candidates(object_ids).await } } @@ -930,6 +949,17 @@ mod tests { GraphExpansionFilteredReason, GraphExpansionLifecyclePolicy, }; + #[tokio::test] + async fn temporary_vector_store_removes_its_directory_on_drop() { + let store = TemporaryVectorCandidateStore::open(2).await; + let path = store.directory.path().to_path_buf(); + assert!(path.exists()); + + drop(store); + + assert!(!path.exists()); + } + #[tokio::test] async fn graph_fake_preserves_objects_links_lifecycle_and_raw_refs() { let store = FakeGraphAuthorityStore::new(); From bc590ba9475fb7be47820c30fc0a1ca26cb20530 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 09:27:24 +0900 Subject: [PATCH 04/14] test: await embedded shard cleanup --- docs/coding-agent/lessons.md | 31 ++++++++++++++++--- .../v0-1-6-embedded-vector-recall-plan.md | 2 +- src/adapters/qdrant_edge/mod.rs | 1 + tests/vector_port_contract_tests.rs | 28 ++++++++++++++++- 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/docs/coding-agent/lessons.md b/docs/coding-agent/lessons.md index 167f47cc..9d2b586a 100644 --- a/docs/coding-agent/lessons.md +++ b/docs/coding-agent/lessons.md @@ -667,17 +667,40 @@ Context: - Roles involved: Worker | Reviewer Symptom: -- The embedded test fixture dropped its temporary directory immediately after the store destructor only signalled the shard-owner thread, so Windows could attempt deletion while shard files remained open and `TempDir` would ignore the cleanup error. +- The embedded test fixture could drop its temporary directory while the shard still held Windows file handles, and `TempDir` would ignore the cleanup error. Root cause: -- The fixture relied on field drop order but did not distinguish a shutdown signal from an acknowledged owner shutdown. +- The fixture first relied on a shutdown signal; the initial correction then relied on a reply that the owner sent after flush but before the `EdgeShard` destructor released its handles. Fix applied: -- The fixture destructor moves the store to a helper thread, awaits the adapter's test-only acknowledged close there, joins the helper, and only then lets the temporary directory drop. +- The owner now drops `EdgeShard` before sending the test-only close reply; the fixture waits for that reply on a helper thread, joins it, and only then lets the temporary directory drop. Prevention: -- A temporary fixture that owns a background resource must wait for that resource's shutdown acknowledgement before filesystem cleanup, and a test must assert the directory is actually removed. +- A shutdown acknowledgment must be emitted after the owned resource's destructor completes, not merely after its final flush; filesystem tests must also assert the directory is actually removed. - Residual risk / waiver: none. Evidence: - Reviewer finding on Task_7 and `temporary_vector_store_removes_its_directory_on_drop`. + +## 2026-09-04 — Explicitly Clean Integration Roots Around Signal-Only Production Drop [tags: review, cleanup, windows, integration-tests, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 review revision +- Roles involved: Worker | Reviewer + +Symptom: +- The restart contract test let a reopened `CharacterMemory` and its `TempDir` fall out of scope together, so the facade's intentionally signal-only production destructor could race directory deletion and leave one `.tmp*` root on Windows. + +Root cause: +- The integration fixture did not own the shutdown-to-cleanup lifecycle explicitly, and sibling embedded cases used the same scope-drop pattern. + +Fix applied: +- All embedded `TempDir` cases in the vector-port integration file now drop the facade, retry root removal with a ten-second bound while the owner releases its handles, and assert the root is gone. + +Prevention: +- Integration tests around signal-only production destructors must keep the temporary path, perform bounded cleanup after dropping the facade, and include a before/after temporary-root census for the full suite. +- Residual risk / waiver: none. + +Evidence: +- Reviewer finding on Task_7 and the full-suite `.tmp*` before/after census recorded in the completed plan. diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index ab817724..4489738b 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -261,7 +261,7 @@ Append-only editing rule (applies to both logs below): when appending an entry, 4. Hint filter semantics: `rg -n "VectorCandidateFilter|CandidateFilter|match_or_unknown|matches_or_unknown|MatchUnknown" src tests` returned zero hits; the query carries only object-type scope, with its empty-scope behavior and predicate rules pinned by ADR-I-0024 and the port contract tests. 5. Evaluation baseline capability: this repository exposes singleton-scoped traced candidates, completeness telemetry, and `max_embedding_surfaces`; `rg -n "SearchPointsBuilder|QDRANT_OBJECT_ID_FIELD|QDRANT_OBJECT_TYPE_FIELD|QDRANT_CONTENT_TEXT_FIELD" crates/cmem-eval-adapter-cmem/src/lib.rs` still finds the companion repository's direct Qdrant baseline, so its trace migration, row-level identity/rank A/B diff, and post-switch zero-hit census are explicitly pending in that repository. - 2026-09-04 Task_7 validation closeout: `cargo fmt --all -- --check` passed; `cargo clippy --all-targets --all-features -- -D warnings` passed on Rust 1.97; bare `cargo test` passed 396 library tests, 31 integration tests, and one doc test with five intentional ignores after the point-identity and temporary-directory-cleanup fixtures were added; the live-switch `cargo test` passed with both service/embedded parity tests executing; and the four ignored service-Qdrant tests passed explicitly under the live endpoint. The package is 0.1.6, the phase and roadmap are finished, and this plan is moved to completed. Independent reviewer approval and stack merge remain Orchestrator gates. -- 2026-09-04 Task_7 review revision: the service scroll-width overflow now returns the typed `Conversion` kind; `rg -n "SURFACE_FIELD" src tests` returns zero hits; completeness docs and ADR-I-0024 define exhaustive recall by proving every scoped record was scored through a known-exhaustive path; and the temporary embedded fixture waits for acknowledged owner shutdown before directory cleanup, with a Windows-sensitive deletion assertion. The service-free suite, strict clippy, strict rustdoc generation, and formatting gate all pass; no live rerun was needed because the delta changes local conversion classification, documentation, and the embedded test fixture only. +- 2026-09-04 Task_7 review revision: the service scroll-width overflow now returns the typed `Conversion` kind; `rg -n "SURFACE_FIELD" src tests` returns zero hits; completeness docs and ADR-I-0024 define exhaustive recall by proving every scoped record was scored through a known-exhaustive path; the test close acknowledgment follows the `EdgeShard` destructor; and every embedded vector-port integration fixture performs bounded root cleanup after dropping its facade. The service-free suite, strict clippy, strict rustdoc generation, and formatting gate all pass; the final full-suite Windows `.tmp*` census was 148 before and 148 after (zero delta). No live rerun was needed because the delta changes local conversion classification, documentation, and embedded test lifecycles only. ## Decision Log (append-only; re-plans and major discoveries) diff --git a/src/adapters/qdrant_edge/mod.rs b/src/adapters/qdrant_edge/mod.rs index aab1b616..e5231a90 100644 --- a/src/adapters/qdrant_edge/mod.rs +++ b/src/adapters/qdrant_edge/mod.rs @@ -357,6 +357,7 @@ fn owner_loop(shard: EdgeShard, commands: mpsc::Receiver) { } Command::Shutdown { reply } => { let result = shard.flush().map_err(edge_error); + drop(shard); if let Some(reply) = reply { let _ = reply.send(result); } diff --git a/tests/vector_port_contract_tests.rs b/tests/vector_port_contract_tests.rs index 0cadc3ec..d8516156 100644 --- a/tests/vector_port_contract_tests.rs +++ b/tests/vector_port_contract_tests.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::{fs, io::ErrorKind, path::Path, time::Duration}; use async_trait::async_trait; use character_memory::{ @@ -52,6 +52,7 @@ async fn embedded_default_contract_is_service_free_restart_safe_and_canonical() drop(memory); let reopened = open_embedded(temp.path(), collection).await.unwrap(); assert_eq!(ids(&episode_snapshot(&reopened).await), vec![id(2), id(3)]); + drop_embedded_and_remove_root(reopened, temp).await; } #[tokio::test] @@ -87,6 +88,7 @@ async fn embedded_zero_norm_contract_rejects_records_and_exhaustively_scores_que .unwrap(); assert_zero_norm_contract(&memory).await; + drop_embedded_and_remove_root(memory, temp).await; } #[tokio::test] @@ -109,6 +111,7 @@ async fn service_and_embedded_share_the_zero_norm_contract() { } .await; + drop_embedded_and_remove_root(embedded, temp).await; test_support::cleanup_collection(&collection).await; result } @@ -142,10 +145,33 @@ async fn service_and_embedded_admit_identical_candidates_in_identical_order() { } .await; + drop_embedded_and_remove_root(embedded, temp).await; test_support::cleanup_collection(&collection).await; result } +async fn drop_embedded_and_remove_root(memory: CharacterMemory, temp: TempDir) { + let path = temp.keep(); + drop(memory); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + match fs::remove_dir_all(&path) { + Ok(()) => break, + Err(error) if error.kind() == ErrorKind::NotFound => break, + Err(_) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(20)).await; + } + Err(error) => panic!( + "embedded vector root {} remained locked after shutdown: {error}", + path.display() + ), + } + } + + assert!(!path.exists()); +} + async fn open_embedded(path: &Path, collection: &str) -> Result { open( common_settings() From 6be4f23b2ddd9ad6f48bb740ab2f92dd8ed81d3a Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 10:02:18 +0900 Subject: [PATCH 05/14] test: close indexed recall fixtures --- docs/coding-agent/lessons.md | 23 +++++++++++++++++++ .../v0-1-6-embedded-vector-recall-plan.md | 1 + src/adapters/qdrant_edge/mod.rs | 7 ++++++ 3 files changed, 31 insertions(+) diff --git a/docs/coding-agent/lessons.md b/docs/coding-agent/lessons.md index 9d2b586a..eb35c71d 100644 --- a/docs/coding-agent/lessons.md +++ b/docs/coding-agent/lessons.md @@ -704,3 +704,26 @@ Prevention: Evidence: - Reviewer finding on Task_7 and the full-suite `.tmp*` before/after census recorded in the completed plan. + +## 2026-09-04 — Close Every Direct Background-Owner Test Fixture Explicitly [tags: review, cleanup, windows, tests, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 review revision +- Roles involved: Worker | Reviewer + +Symptom: +- The service-up full suite left one 112 MB `.tmp*` root named `indexed` after the indexed exact-recall adapter test. + +Root cause: +- The earlier cleanup audit covered the shared fixture and integration facade cases but missed direct adapter stores whose background owners still held the temporary directory when it dropped. + +Fix applied: +- The indexed exact-recall test now awaits both direct-store closes before explicitly closing and asserting removal of its temporary root; the remaining direct point-identity and zero-norm stores also close explicitly. + +Prevention: +- Audit every `TempDir`/store pair, require each background owner to acknowledge close before its directory is removed, and compare exact temporary-root counts around both bare and live-switch full suites. +- Residual risk / waiver: none. + +Evidence: +- Reviewer item 6; the bare full-suite census held at 149 before and after, and the `REQUIRE_QDRANT_TESTS=1` full-suite census held at 149 before and after. diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 4489738b..0ddc94cd 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -262,6 +262,7 @@ Append-only editing rule (applies to both logs below): when appending an entry, 5. Evaluation baseline capability: this repository exposes singleton-scoped traced candidates, completeness telemetry, and `max_embedding_surfaces`; `rg -n "SearchPointsBuilder|QDRANT_OBJECT_ID_FIELD|QDRANT_OBJECT_TYPE_FIELD|QDRANT_CONTENT_TEXT_FIELD" crates/cmem-eval-adapter-cmem/src/lib.rs` still finds the companion repository's direct Qdrant baseline, so its trace migration, row-level identity/rank A/B diff, and post-switch zero-hit census are explicitly pending in that repository. - 2026-09-04 Task_7 validation closeout: `cargo fmt --all -- --check` passed; `cargo clippy --all-targets --all-features -- -D warnings` passed on Rust 1.97; bare `cargo test` passed 396 library tests, 31 integration tests, and one doc test with five intentional ignores after the point-identity and temporary-directory-cleanup fixtures were added; the live-switch `cargo test` passed with both service/embedded parity tests executing; and the four ignored service-Qdrant tests passed explicitly under the live endpoint. The package is 0.1.6, the phase and roadmap are finished, and this plan is moved to completed. Independent reviewer approval and stack merge remain Orchestrator gates. - 2026-09-04 Task_7 review revision: the service scroll-width overflow now returns the typed `Conversion` kind; `rg -n "SURFACE_FIELD" src tests` returns zero hits; completeness docs and ADR-I-0024 define exhaustive recall by proving every scoped record was scored through a known-exhaustive path; the test close acknowledgment follows the `EdgeShard` destructor; and every embedded vector-port integration fixture performs bounded root cleanup after dropping its facade. The service-free suite, strict clippy, strict rustdoc generation, and formatting gate all pass; the final full-suite Windows `.tmp*` census was 148 before and 148 after (zero delta). No live rerun was needed because the delta changes local conversion classification, documentation, and embedded test lifecycles only. +- 2026-09-04 Task_7 service-up cleanup revision: the indexed exact-recall canary awaits both direct-store closes before removing and asserting its temporary root, and the remaining direct point-identity and zero-norm adapter stores also close explicitly. The focused canary, bare full suite, service-up full suite with `REQUIRE_QDRANT_TESTS=1`, strict clippy, strict rustdoc generation, and formatting gate pass; the final Windows `.tmp*` censuses were 149 before and 149 after for both full-suite modes (zero delta). ## Decision Log (append-only; re-plans and major discoveries) diff --git a/src/adapters/qdrant_edge/mod.rs b/src/adapters/qdrant_edge/mod.rs index e5231a90..4e5f1ef2 100644 --- a/src/adapters/qdrant_edge/mod.rs +++ b/src/adapters/qdrant_edge/mod.rs @@ -776,6 +776,7 @@ mod tests { .unwrap(); assert_eq!(point.id.to_string(), qdrant_point_id(&record).to_string()); + store.close().await.unwrap(); } async fn upsert( @@ -921,6 +922,7 @@ mod tests { result.completeness, VectorRecallCompleteness::Exhaustive { scanned: 4 } ); + store.close().await.unwrap(); } #[tokio::test] @@ -1114,6 +1116,7 @@ mod tests { #[tokio::test] async fn indexed_test_configuration_reports_boundary_and_matches_exact_recall() { let temp = TempDir::new().unwrap(); + let path = temp.path().to_path_buf(); let (records, embeddings) = records(200, &[1.0, 0.0]); let exact = QdrantEdgeVectorCandidateStore::open(temp.path(), "exact", 2) .await @@ -1142,6 +1145,10 @@ mod tests { indexed_result.completeness, VectorRecallCompleteness::BoundaryTieClosed { .. } )); + exact.close().await.unwrap(); + indexed.close().await.unwrap(); + temp.close().unwrap(); + assert!(!path.exists()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 0f3e90e322953f481f3a24e7cada54b177863b89 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 16:27:06 +0900 Subject: [PATCH 06/14] fix: align vector recall accounting and ADRs --- docs/coding-agent/lessons.md | 48 ++++++++- .../v0-1-6-embedded-vector-recall-plan.md | 25 +++-- docs/coding-agent/rules/orchestrator.md | 2 +- ...dded-qdrant-edge-vector-candidate-store.md | 4 +- ...ness-and-prefilters-never-match-unknown.md | 48 ++++----- ...I-0025-vector-record-is-a-read-contract.md | 20 ++-- ...ctor-baselines-read-the-retrieval-trace.md | 2 +- ...blocking-owner-that-flushes-every-write.md | 4 +- ...current-columns-and-never-match-unknown.md | 101 ++++++++++++++++++ docs/design/database/vector_payload_design.md | 4 +- ...v0_1_6_embedded_vector_candidate_recall.md | 14 +-- docs/roadmap/development_roadmap.md | 4 +- src/adapters/qdrant/store.rs | 50 ++------- src/api/types/retrieval.rs | 13 ++- src/config/app_settings.rs | 5 +- src/ports/vector_candidate.rs | 6 +- 16 files changed, 235 insertions(+), 115 deletions(-) create mode 100644 docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md diff --git a/docs/coding-agent/lessons.md b/docs/coding-agent/lessons.md index eb35c71d..90f688a6 100644 --- a/docs/coding-agent/lessons.md +++ b/docs/coding-agent/lessons.md @@ -650,7 +650,7 @@ Root cause: - Documentation generalized one implementation path into the verdict's semantic condition instead of checking every branch that produces the public enum. Fix applied: -- State that exhaustive means every requested-scope record was scored through a known-exhaustive path, the cutoff cohort closed, and `scanned` came from the scope count; name unindexed scans and full-scope scrolls as examples. +- State that exhaustive means every requested-scope record was scored through a known-exhaustive path, the cutoff cohort closed, and `scanned` came from the records that path actually scored; name unindexed scans and full-scope scrolls as examples. Prevention: - Define public verdicts from observable proof conditions and audit every constructor branch before documenting implementation examples. @@ -727,3 +727,49 @@ Prevention: Evidence: - Reviewer item 6; the bare full-suite census held at 149 before and after, and the `REQUIRE_QDRANT_TESTS=1` full-suite census held at 149 before and after. + +## 2026-09-04 — Derive Exhaustive Counters From The Records Actually Scored [tags: review, concurrency, telemetry, vector-recall, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 Copilot follow-up +- Roles involved: Worker | Reviewer | Orchestrator + +Symptom: +- The service zero-norm path used a separate filtered count request for `scanned`, so a concurrent write between that request and the full-scope scroll could make the verdict report a population different from the records it scored. + +Root cause: +- The counter was treated as a scope snapshot instead of evidence about the exhaustive operation that produced the candidates. + +Fix applied: +- Derive `scanned` from the final closed scroll response, which is the exact scoped record set scored by the zero-norm path. + +Prevention: +- A telemetry counter describing completed work must come from that operation's result, not a separate query that can observe different state. +- Residual risk / waiver: none. + +Evidence: +- Copilot finding on PR #79 and the service zero-norm live regression. + +## 2026-09-04 — Keep One Governing Claim Per Decision Record [tags: review, adr, durable-docs, worker] + +Context: +- Plan: `docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md` +- Task/Wave: Task_7 / Wave 5 Copilot follow-up +- Roles involved: Worker | Reviewer | Orchestrator + +Symptom: +- ADR-I-0024 combined recall-completeness semantics with vector-prefilter admission, and the decision set retained unanchored time-relative wording. + +Root cause: +- Two nearby port concerns were recorded together without applying the one-decision warrant test or the durable-wording sweep independently to each claim. + +Fix applied: +- Keep completeness in ADR-I-0024, move prefilter admission into ADR-I-0028 with its own warrant and revisit conditions, and remove unanchored time-relative wording from ADR-I-0023 through ADR-I-0028. + +Prevention: +- Before finalising an ADR cluster, state one governing claim per record and run a time-relative-word census across every record in the cluster. +- Residual risk / waiver: none. + +Evidence: +- Copilot findings on planning PR #72 and the post-split ADR census. diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 0ddc94cd..f112e9f7 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -6,14 +6,14 @@ - work_type: mixed ## Goal -- Deliver the phase described in `docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md` under ADR-I-0023 through ADR-I-0027: a redesigned vector port contract, a five-field vector record, an embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) as the default vector mode with the service adapter retained as the explicit service mode, and a shared contract suite over both adapters; the evaluation repository's move of its vector-only baseline onto the retrieval trace is planned there and consumed here as closeout evidence. +- Deliver the phase described in `docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md` under ADR-I-0023 through ADR-I-0028: a redesigned vector port contract, a five-field vector record, an embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) as the default vector mode with the service adapter retained as the explicit service mode, and a shared contract suite over both adapters; the evaluation repository's move of its vector-only baseline onto the retrieval trace is planned there and consumed here as closeout evidence. ## Definition of Done - Every acceptance criterion in the phase document's "Acceptance criteria" section holds with recorded evidence. - Every row of the phase document's deferral-reconfirmation checklist has its evidence produced and cited in the Progress Log. - Every deletion listed under "Deletions that are deliverables" is gone, with a zero-hit census. - Both repositories' service-gated suites execute (not skip) under the service-backed CI job. -- One PR per wave in this repository, merged by the decider; the evaluation repository's cross-mode comparison is available as consumed evidence at closeout. +- One or more PRs per wave in this repository, stacked on the planning PR and merged by the decider; the evaluation repository's cross-mode comparison is available as consumed evidence at closeout. ## Scope / Non-goals - Scope: the phase document's deliverables and deletions, all in this repository. @@ -174,12 +174,17 @@ - type: chore - owns: - src/test_support.rs - - src/**/tests (fake stores only) + - src/memory.rs (inline fake-store tests only) + - src/models/vector.rs (fake-only re-export) + - src/models/vector/candidate_record.rs (fake type retirement and inline tests) + - src/models/vector/record.rs (fake-only conversions and inline tests) + - src/policy.rs (fake-only test re-export) + - src/usecases/correct_forget.rs (inline fake-store tests only) + - src/usecases/retrieve.rs (inline fake-store tests only) - src/api/types/retrieval.rs (doc comments only) - src/ports/vector_candidate.rs (doc comments only) - src/composition.rs, src/config/app_settings.rs (public constructor doc comments only; deferred from the Task_4 Copilot review) - src/adapters/qdrant/store.rs, src/adapters/qdrant/payload.rs, src/adapters/qdrant_edge/mod.rs (Task_8 audit dispositions only) - - src/models/vector/candidate_record.rs (header comment only) - tests/vector_port_contract_tests.rs, tests/support/** (point-id parity fixture and fake-retirement fallout) - README.md (stale filtering line) - docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md (closeout status) @@ -194,8 +199,8 @@ - acceptance: - Zero-hit census for the retired fake and record type. - All five checklist rows cite evidence in the Progress Log. - - Task_8 audit dispositions landed: one point-identity derivation (the v5 derivation) shared by both adapters with a parity assertion and no compat shim; one shared read function for the record contract returning the closed error vocabulary, with the service adapter's stringly database errors for payload and scroll-limit faults removed; the canary pins that an indexed shard searched exactly returns the exhaustive result; the service zero-norm verdict's `scanned` comes from the scope count or the loop invariant is stated; the duplicate vector-config validation is reduced to one or justified in place; the stale header comments and README line are corrected; the public facade constructor and `Settings::new` rustdoc describe both vector-store modes in backend-neutral terms (embedded default with a store path; explicit service mode with a connection string). - - The public completeness type documents all four verdict meanings and the `scanned` and `fetched` counters, stating that a closed boundary verdict is deterministic only for the index-returned prefix, never population-level completeness; the port trait's doc comment states the verdict guarantees (tie closure through the shared loop; exhaustive only when every scoped record was scored through a path the adapter knows to be exhaustive, including an unindexed scan or a full-scope scroll, with a closed cohort and `scanned` from the scope count; open at the bound) so a future adapter cannot label an index-produced prefix exhaustive (deferred from the Task_2 review to avoid re-cascading the stack mid-wave). + - Task_8 audit dispositions landed: one point-identity derivation (the v5 derivation) shared by both adapters with a parity assertion and no compat shim; one shared read function for the record contract returning the closed error vocabulary, with the service adapter's stringly database errors for payload and scroll-limit faults removed; the canary pins that an indexed shard searched exactly returns the exhaustive result; the service zero-norm verdict's `scanned` is the size of the final scroll response whose records were actually scored; the duplicate vector-config validation is reduced to one or justified in place; the stale header comments and README line are corrected; the public facade constructor and `Settings::new` rustdoc describe both vector-store modes in backend-neutral terms (embedded default with a store path; explicit service mode with a connection string). + - The public completeness type documents all four verdict meanings and the `scanned` and `fetched` counters, stating that `scanned` is the number of scoped records actually scored and `fetched` is the final prefix size rather than a cumulative count, and that a closed boundary verdict is deterministic only for the index-returned prefix, never population-level completeness; the port trait's doc comment states the verdict guarantees (tie closure through the shared loop; exhaustive only when every scoped record was scored through a path the adapter knows to be exhaustive, including an unindexed scan or a full-scope scroll, with a closed cohort; open at the bound) so a future adapter cannot label an index-produced prefix exhaustive (deferred from the Task_2 review to avoid re-cascading the stack mid-wave). - validation: - kind: command required: true @@ -258,11 +263,12 @@ Append-only editing rule (applies to both logs below): when appending an entry, 1. Canonical-candidates newtype survival: `rg -n "CanonicalCandidates" src tests` shows the newtype remains the `VectorCandidateRecall.candidates` envelope field and is constructed only at adapter/tie-closure and deliberate test-double boundaries; `canonical_candidates_dedupe_identity_at_highest_score_and_totally_order_ties` remains in the service-free suite. 2. Dual text columns: `rg -n "content_text|embedding_text|CONTENT_TEXT|EMBEDDING_TEXT" src tests` shows no `content_text` column in this repository and only the required five-field `embedding_text` record/provenance path. The companion repository still has its direct Qdrant `content_text` reader, so its before/after vector-only identity-and-text A/B evidence is explicitly pending in that repository. 3. Search completeness: `backend_fetch_cap_reports_an_open_boundary_without_allocating_rows`, `retrieval_telemetry_preserves_every_vector_recall_completeness_verdict`, the live service boundary tests, and `service_and_embedded_admit_identical_candidates_in_identical_order` cover open, telemetry, closed, and cross-adapter exhaustive-versus-closed behavior through the shared tie loop. - 4. Hint filter semantics: `rg -n "VectorCandidateFilter|CandidateFilter|match_or_unknown|matches_or_unknown|MatchUnknown" src tests` returned zero hits; the query carries only object-type scope, with its empty-scope behavior and predicate rules pinned by ADR-I-0024 and the port contract tests. + 4. Hint filter semantics: `rg -n "VectorCandidateFilter|CandidateFilter|match_or_unknown|matches_or_unknown|MatchUnknown" src tests` returned zero hits; the query carries only object-type scope, with its empty-scope behavior pinned by ADR-I-0024, its predicate rules pinned by ADR-I-0028, and its behavior covered by the port contract tests. 5. Evaluation baseline capability: this repository exposes singleton-scoped traced candidates, completeness telemetry, and `max_embedding_surfaces`; `rg -n "SearchPointsBuilder|QDRANT_OBJECT_ID_FIELD|QDRANT_OBJECT_TYPE_FIELD|QDRANT_CONTENT_TEXT_FIELD" crates/cmem-eval-adapter-cmem/src/lib.rs` still finds the companion repository's direct Qdrant baseline, so its trace migration, row-level identity/rank A/B diff, and post-switch zero-hit census are explicitly pending in that repository. - 2026-09-04 Task_7 validation closeout: `cargo fmt --all -- --check` passed; `cargo clippy --all-targets --all-features -- -D warnings` passed on Rust 1.97; bare `cargo test` passed 396 library tests, 31 integration tests, and one doc test with five intentional ignores after the point-identity and temporary-directory-cleanup fixtures were added; the live-switch `cargo test` passed with both service/embedded parity tests executing; and the four ignored service-Qdrant tests passed explicitly under the live endpoint. The package is 0.1.6, the phase and roadmap are finished, and this plan is moved to completed. Independent reviewer approval and stack merge remain Orchestrator gates. - 2026-09-04 Task_7 review revision: the service scroll-width overflow now returns the typed `Conversion` kind; `rg -n "SURFACE_FIELD" src tests` returns zero hits; completeness docs and ADR-I-0024 define exhaustive recall by proving every scoped record was scored through a known-exhaustive path; the test close acknowledgment follows the `EdgeShard` destructor; and every embedded vector-port integration fixture performs bounded root cleanup after dropping its facade. The service-free suite, strict clippy, strict rustdoc generation, and formatting gate all pass; the final full-suite Windows `.tmp*` census was 148 before and 148 after (zero delta). No live rerun was needed because the delta changes local conversion classification, documentation, and embedded test lifecycles only. - 2026-09-04 Task_7 service-up cleanup revision: the indexed exact-recall canary awaits both direct-store closes before removing and asserting its temporary root, and the remaining direct point-identity and zero-norm adapter stores also close explicitly. The focused canary, bare full suite, service-up full suite with `REQUIRE_QDRANT_TESTS=1`, strict clippy, strict rustdoc generation, and formatting gate pass; the final Windows `.tmp*` censuses were 149 before and 149 after for both full-suite modes (zero delta). +- 2026-09-04 Task_7 telemetry and decision-record follow-up: the service zero-norm path derives `scanned` from the final scroll response whose scoped records it scored, with no concurrent count request; `fetched` is documented as the final prefix size rather than a cumulative total. ADR-I-0024 now governs completeness only, while accepted ADR-I-0028 governs prefilters and requires fully populated or backfilled current columns before activation. `Settings::new` documents that facade construction consumes and validates the retained vector location, and the Orchestrator rule metadata carries the review date. The ADR-I-0023 through ADR-I-0028 temporal-word census and stale-reference census returned zero hits. Formatting, strict all-target/all-feature clippy, and strict all-feature rustdoc passed. The final bare and service-up full suites each passed 396 library tests, 31 integration tests, and one doc test with five intentional ignores; the changed ignored live zero-norm regression also passed explicitly. The recursive `.tmp*` census stayed 4 before and 4 after the final bare run and 0 before and 0 after the service-up run. Two earlier bare runs exposed unrelated load-sensitive SQLite and Edge lock timing failures; each exact rerun passed, followed by the clean full-suite run recorded here. ## Decision Log (append-only; re-plans and major discoveries) @@ -289,6 +295,11 @@ Append-only editing rule (applies to both logs below): when appending an entry, - The blocking-owner and per-write-flush rules moved out of ADR-I-0023 into ADR-I-0027 as their own decision. - ADR-I-0024 narrowed to the completeness verdict and the prefilter rule (unknown never matches); the scope-only query is a current state, the type shape is a non-binding appendix, and the two predicate paths are notes binding on no phase. - 2026-09-03 — Merge shape: the records and plan are not merged ahead of the implementation, because ADR-I-0023's embedded default rests on evidence the implementation produces. Implementation waves land as PRs against the planning branch (`plan/v0-1-6-embedded-vector-recall`), which stays the readable home of the records; the phase merges to main as one change once the decision is solidified by its evidence. +- 2026-09-04 — ADR-I-0024 split by governing claim. + - Trigger / new insight: completeness evidence and prefilter admission have different warrants, failure modes, and revisit triggers; keeping both in one record obscured which rule governed a change. + - Plan delta: ADR-I-0024 now decides completeness verdicts and counters only; accepted ADR-I-0028 decides that prefilters never match unknown and may activate only after their current columns are fully populated or backfilled. Phase, payload, roadmap, and ADR cross-references follow that boundary. + - Tradeoff: one additional decision record and explicit dependency links in exchange for independent amendment and auditability. + - User approval: yes, 2026-09-04. ## Notes - Risks: the row/summary schema move in the evaluation repository (typed backend identity) is a clean break under its compatibility policy and must not touch sealed evidence; the latency guidance and the stripped dependency weight must be measured, not assumed; the engine is beta, so its pin is exact and its bump is gated by the canary. diff --git a/docs/coding-agent/rules/orchestrator.md b/docs/coding-agent/rules/orchestrator.md index a9681518..fffcf169 100644 --- a/docs/coding-agent/rules/orchestrator.md +++ b/docs/coding-agent/rules/orchestrator.md @@ -2,7 +2,7 @@ rule_schema_version: 2 suite_id: "rules-cm-20260719" rule_file: "orchestrator" -last_updated: "2026-07-23" +last_updated: "2026-09-04" --- # Orchestrator Repository Rules diff --git a/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md b/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md index 9e8ae070..99ddf0f6 100644 --- a/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md +++ b/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md @@ -89,7 +89,7 @@ It is the only candidate that offers, in one engine, the capabilities the decade ### Rejected Alternatives -Option 2 (in-house exact scan) violates the library-over-in-house driver: the library would own distance computation, blob encoding, and scan scheduling, and every capability the decade standard needs (index, quantization, named vectors) would have to be written or migrated to later; it is rejected outright, not deferred. +Option 2 (in-house exact scan) violates the library-over-in-house driver: the library would own distance computation, blob encoding, and scan scheduling, and every capability the decade standard needs (index, quantization, named vectors) would require independent implementation or migration; it is rejected outright, not deferred. Option 3 (sqlite-vec 0.1.9) measured well — builds on Rust 1.95 and 1.97, 19 additional tree lines, about 6.2 MB unstripped, deterministic ties across fresh files and processes, parity five of five with score delta at or below 1.6e-7 — but its stable release is exhaustive-only with approximate indexing existing only in a pre-release, it carries limits on dimensions, result count, and metadata columns, and it is a pre-1.0 binding with a single maintainer; at decade scale its future is a second migration, so it is rejected for this role and reopened only if the chosen engine fails its Revisit When triggers and the extension has shipped a stable approximate index. Option 4 (LanceDB) was rejected on dependency weight relative to the chosen engine without a spike, since the chosen engine covers its capabilities; it is reopened only alongside Option 3's reopening. Option 5 fails restart safety, which the persistent-graph-authority phase made a requirement for every store that survives a process; rejected outright. @@ -98,7 +98,7 @@ Option 6 defers a decision whose deciding evidence the same change produces: the ## Consequences - Positive: a fully self-contained local deployment exists; the default test path needs no running service; both adapters are held to one contract by one suite; the embedded store can grow into indexed, quantized, and memory-mapped operation without an engine switch. -- Positive: shared engine family means payload and filter conventions are written once and the service parity result (score delta 0.0) is structural, not coincidental. +- Positive: the shared engine family gives both adapters one payload and filter convention, and the service parity result (score delta 0.0) is structural, not coincidental. - Negative / tradeoffs: the engine is beta and its API may change; the canary test and the pinned version turn that into a build-time failure rather than a runtime one. - Negative / tradeoffs: about 30.5 MB of unstripped binary and a higher toolchain floor; the weight deliverable exists to establish the real number. - Negative / tradeoffs: two adapters must be kept in parity for every port change; the parity suite is the cost of that guarantee. diff --git a/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md b/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md index 2752e5aa..71f162e9 100644 --- a/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md +++ b/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md @@ -6,10 +6,10 @@ deciders: ["ebigunso"] consulted: ["Claude Fable 5.1"] informed: [] warrant: - warranted_by: "without this record, future work would likely add a vector-layer predicate as a three-valued hint filter that matches unknown values, or let an adapter truncate an unclosed equal-score cohort without saying so, because both are the natural first implementation and both have already happened in this repository" - detected_signals: "cross-boundary contract shape (port postcondition) with tempting alternatives; rejected alternative likely to be re-proposed; premises likely to expire (no vector-layer predicate has a caller, and stored values are not guaranteed present)" - cost_of_violation: "a prefilter that matches unknown values admits stale candidates that graph verification then silently discards, and an unreported open cohort makes top-K membership vary between runs — both surface as unexplained retrieval nondeterminism in evaluation evidence long after the cause is forgotten" - cost_of_wrong_preservation: "if the unknown-never-matches rule is preserved after every stored value is guaranteed present and synchronised, adapters carry a defensive arm for a case that cannot occur" + warranted_by: "without this record, future work would likely let an adapter truncate an unclosed equal-score cohort without reporting that loss of determinacy, because the service adapter did so before ADR-I-0024" + detected_signals: "cross-boundary contract shape (port postcondition) with tempting alternatives; rejected alternative likely to be re-proposed; premises likely to expire when an adapter cannot classify its own cutoff" + cost_of_violation: "an unreported open cohort makes top-K membership vary between runs and surfaces as unexplained retrieval nondeterminism in evaluation evidence after the cause is forgotten" + cost_of_wrong_preservation: "if an adapter cannot distinguish an exhaustive population from an index-produced prefix, preserving the four-way vocabulary without an unknown verdict would force false precision" cost_of_over_extension: "treating the completeness verdict as an error condition would fail retrieval on a determinism caveat about non-authoritative candidates" depends_on: [implementation/ADR-I-0018-responsibility-boundary-modules-with-enforced-dependency-direction.md, implementation/ADR-I-0022-retain-measured-retrieval-defaults.md] implements: [] @@ -18,38 +18,32 @@ superseded_by: null supersession_scope: null --- -# ADR-I-0024: Vector candidate recall reports its completeness, and a vector-layer prefilter never matches an unknown value +# ADR-I-0024: Vector candidate recall reports its completeness ## Context and Problem Statement The vector candidate port promised deterministic admission: at most `limit` unique object-and-surface matches in canonical order, with equal-score cohorts at the cutoff closed before truncation (ADR-I-0022 records the fix). -The service adapter closes the cohort by growing its fetch up to a bound, but when the bound is hit it returns the truncated set with no signal, so a caller cannot tell "this top-K is determinate" from "membership may vary between runs", and evaluation evidence later attributes the variation to retrieval. -Separately, the port once carried currentness predicates implemented as match-or-unknown: a record whose payload lacked the field satisfied a positive predicate in both the service adapter and the test fake, so the filter admitted records under a rationale ("current") that was not true of them; and because the field was written only at upsert, a record whose value had since changed in graph authority was filtered on a stale value, admitted when it should not have been or excluded when it should have been returned. -Those filters were deleted as speculative when no caller used them. -A second adapter (ADR-I-0023) makes both gaps matter: below its indexing threshold an embedded shard scans exhaustively and needs a way to say so, and two adapters must agree on what a prefilter may do. +The service adapter closes the cohort by growing its fetch up to a bound, but when the bound is hit it returns the truncated set with no signal, so a caller cannot tell "this top-K is determinate" from "membership may vary between runs", and subsequent evaluation evidence attributes the variation to retrieval. +A second adapter (ADR-I-0023) makes the gap matter in another direction: below its indexing threshold an embedded shard scans exhaustively and needs a way to report population-level determinacy. ## Decision Drivers - Retrieval rationale must be inspectable: the philosophy asks that a developer can see why a memory was or was not retrieved, and an unreported open cohort is unexplained recall. - Candidate recall is non-authoritative; graph authority verifies every candidate, so a determinism caveat must never become a retrieval failure. -- A prefilter false negative is a memory that silently never returns — a continuity loss nobody can inspect — while a false positive costs one root slot and is discarded by verification; the asymmetry decides what a vector-layer predicate may do. - Each port owns its stated postconditions; upper layers never repair lower-layer output. ## Decision The search result carries, beside the canonical candidates, a completeness verdict stated by the adapter. The verdict distinguishes four situations: no search was issued because the limit was zero or the scope was empty; every stored record in scope was scored, so the requested top-K is determinate over the population; an index answered with a prefix whose cutoff cohort was closed, so the returned set is determinate for that index state although an approximate index may have omitted records it never surfaced; and the overfetch bound was reached with the cutoff cohort still open, so membership may vary. -Adapters state the verdict truthfully: exhaustive only when every record in the requested scope was scored through a path the adapter knows to be exhaustive (an unindexed scan or a full-scope scroll such as the zero-norm path) and the cutoff cohort was closed, with the scanned count taken from the scope rather than the rows returned; an exhaustive path whose cohort stays open at the bound reports open. +Adapters state the verdict truthfully: exhaustive only when every record in the requested scope was scored through a path the adapter knows to be exhaustive (an unindexed scan or a full-scope scroll such as the zero-norm path) and the cutoff cohort was closed; `scanned` is the number of scoped records actually scored, never a truncated prefix. An exhaustive path whose cohort stays open at the bound reports open. The retrieval pipeline records the verdict in retrieval telemetry beside the returned candidate count and never repairs, retries, or fails on it. Degenerate vectors are defined on both sides of the port so that neither adapter has an undefined path: a zero-norm record embedding is rejected at indexing as a typed per-record failure before any adapter sees it, and a zero-norm query scores every candidate zero and reports a truthful verdict. -A vector-layer predicate may be evaluated only over a column that is fully populated for every searchable record (backfilled from graph authority before the predicate is enabled, per ADR-I-0025) and that is immutable or synchronised on every mutation; under those two conditions a missing or unknown value is a defect, not a state, and it never satisfies a positive predicate, so the rule produces no false negative on a correctly populated column and turns an incorrectly populated one into a visible failure rather than a silent widening. -A predicate that needs a stored value the write paths do not keep current, or that is not populated for every record, is not a prefilter; it is a graph-authority question. - ## Character Memory Relevance -Recall that silently varies between runs, or that admits and excludes memories on values nobody keeps true, is the unexplained recall the philosophy forbids: a character that forgets an episode because a stale column excluded it looks like a character that never lived it, and a filter that admits on a blank gives a rationale that is false. -The verdict keeps determinism inspectable; the prefilter rule keeps a candidate stage from being the reason a memory is unreachable, and keeps every stated filter rationale true. +Recall that silently varies between runs is the unexplained recall the philosophy forbids: a character that returns a different member of an equal-score cohort without disclosing the open boundary appears inconsistent for no inspectable reason. +The verdict keeps population-level and prefix-level determinacy visible without making non-authoritative candidate recall a failure. ## Implementation Impact @@ -59,11 +53,10 @@ The verdict keeps determinism inspectable; the prefilter rule keeps a candidate ## Considered Options -1. A completeness verdict beside the candidates, plus the prefilter rule. -2. Silent degradation at the fetch bound (as built). +1. A completeness verdict beside the candidates. +2. Silent degradation at the fetch bound (as built before this record). 3. A boolean complete flag. 4. Fail with an error when the cohort is open at the bound. -5. Resurrect the deleted hint filters for the embedded adapter, which can evaluate them exactly. ## Decision Outcome @@ -72,44 +65,39 @@ It makes the postcondition expressible by the layer that owns it, distinguishes ### Rejected Alternatives -Option 2 hides a determinism caveat that evaluation evidence later attributes to retrieval; rejected outright. +Option 2 hides a determinism caveat that subsequent evaluation evidence attributes to retrieval; rejected outright. Option 3 loses the exhaustive-versus-closed distinction that tells a caller whether population-level determinacy was achieved; rejected outright. Option 4 fails retrieval on a caveat about non-authoritative candidates that graph authority verifies anyway; rejected outright. -Option 5 recreates a prefilter over values that only the upsert path wrote, which the rule above forbids; a predicate is admitted when its stored value is kept in sync or is immutable. ## Consequences - Positive: top-K determinism is observable per retrieval and per adapter. -- Positive: any future prefilter has one admission test — is the column populated for every record and always current — instead of a case-by-case argument. -- Negative / tradeoffs: callers wanting a scoped or time-bounded semantic search wait for a synchronised or immutable column rather than filtering on what happens to be stored. +- Negative / tradeoffs: the verdict exposes a backend boundary that consumers may need to retain in telemetry even when they do not act on it. ## Decision Boundary -Invariant: the search result carries a completeness verdict stated truthfully by the adapter, and the pipeline never repairs, retries, or fails on it; a vector-layer predicate reads only a fully populated column that is immutable or synchronised, and an unknown value never satisfies a positive predicate. +Invariant: the search result carries a completeness verdict stated truthfully by the adapter, and the pipeline never repairs, retries, or fails on it. -Not covered: the current query shape (an embedding, a limit, and an object-type scope, with an empty scope selecting zero — a current state, not a rule; the zero-norm rules above are the one query-shape behaviour this record fixes), the verdict's type and wire shape (the appendix is a reference, not a contract), the telemetry field name, and the service adapter's overfetch bound. +Not covered: the query shape established for v0.1.6, the verdict's type and wire shape (the appendix is a reference, not a contract), the telemetry field name, the service adapter's overfetch bound, and vector-layer prefilter admission (ADR-I-0028). ## Validation - Unit tests on the service adapter's fetch decision assert the closed and open verdicts, including the all-tied cohort at the bound. - A retrieval test asserts the telemetry verdict for each situation using the fakes. - The parity suite asserts exhaustive for the embedded adapter below its indexing threshold and closed for the service adapter on the identical-vector tie fixture; the zero-norm parity fixture asserts exhaustive for both adapters because each scores the full requested scope. -- A census of the vector adapters shows no match-or-unknown condition. - Parity fixtures cover the zero-norm query and the rejected zero-norm record in both adapters. ## Revisit When - An adapter appears that cannot classify its own cutoff (a remote index without a fetch count) — the verdict may need an "unknown" situation, which must still never be treated as an error. -- Every stored value a predicate could read is guaranteed present and synchronised — the unknown arm becomes unreachable and may be removed. ## Consultation impact -Question asked: whether the deleted hint filters should return for the embedded adapter; ruling adopted the prefilter rule instead. Revised 2026-09-03 on the decider's review: the type shape is an appendix and the scope-only query is recorded as current state, not as a rule. +Question asked: how both adapters should expose an unclosed cutoff and an exhaustive population; the ruling adopted a typed verdict that remains telemetry rather than control flow. Revised 2026-09-03 on the decider's review: the type shape is an appendix and the scope-only query is recorded as the v0.1.6 state, not as a rule. Revised 2026-09-04 to separate the prefilter decision into ADR-I-0028. ## More Information -- ADR-I-0022 (tie-cohort closure and canonical ordering, the postcondition this record makes expressible); ADR-I-0023 (the embedded adapter); ADR-I-0025 (the stored record a predicate would extend); ADR-I-0026 (the evaluation reader of the verdict). -- Candidate predicates that satisfy the rule, noted for whichever phase needs them and binding on none: a scope id written at upsert and kept in sync by the link and reflection write paths (scoped continuity); an immutable time window over creation and observation time (a time-bounded retrieval route). +- ADR-I-0022 (tie-cohort closure and canonical ordering, the postcondition this record makes expressible); ADR-I-0023 (the embedded adapter); ADR-I-0025 (the stored vector record); ADR-I-0026 (the evaluation reader of the verdict); ADR-I-0028 (vector-layer prefilter admission). ## Appendix: reference shape (non-binding) diff --git a/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md b/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md index 0ade46f9..764f1eb1 100644 --- a/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md +++ b/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md @@ -6,7 +6,7 @@ deciders: ["ebigunso"] consulted: ["Claude Fable 5.1"] informed: [] warrant: - warranted_by: "without this record, future work would likely re-add relationship, lifecycle, time, or readable-text columns to the vector record because the earlier payload design lists them as intended, or delete the embedded-text column as unread once surfaces become generated" + warranted_by: "without this record, future work would likely re-add relationship, lifecycle, time, or readable-text columns to the vector record because the earlier payload design lists them as intended, or delete the embedded-text column as unread when surfaces are generated" detected_signals: "cross-boundary contract shape (two adapters mirror one record); rejected alternative likely to be re-proposed; premises likely to expire (a retrieval route may need a prefilter); cross-repository obligation (the evaluation baseline read a payload column)" cost_of_violation: "every column that returns without a reader is mirrored across two adapters, indexed at every collection initialisation, and carried stale by write paths that never update it; a column deleted as unread would erase the only record of what a generated vector embedded" cost_of_wrong_preservation: "if a retrieval route needs a prefilter and the five-column rule is preserved as prohibition rather than current state, the predicate is blocked instead of landing through the named re-entry path" @@ -49,7 +49,7 @@ There is no readable-text column (`content_text`). The relationship refs (episode, observation, thread, entity, participant, speaker, supersedes), the lifecycle and currentness flags, the time hints, the ranking and salience hints, the object-specific hints, the graph URI, and the raw source reference are not part of the vector record. Dropping the graph URI partially supersedes ADR-I-0001's clause that every vector payload carries it: the stable object id remains the cross-store identity and the graph URI is derived from it by graph authority, so the pointer was a redundant copy of the id; ADR-I-0001's stable-id decision itself is unchanged. The typed field manifest is the single source of both adapters' column sets and holds exactly the five entries. -ADR-I-0024 rules that a predicate reads only synchronised or immutable values and notes the two candidate predicates (a synchronised scope id; an immutable time window over `created_at` and `observed_at` backfilled from graph authority), so a returning column arrives with its predicate and its reader. +ADR-I-0028 rules that a predicate reads only a fully populated, synchronised or immutable value and notes the two candidate predicates (a synchronised scope id; an immutable time window over `created_at` and `observed_at` backfilled from graph authority), so a returning column arrives with its predicate and its reader. ## Implementation Impact @@ -61,7 +61,7 @@ ADR-I-0024 rules that a predicate reads only synchronised or immutable values an ## Considered Options -1. Five-column read contract; keep `embedding_text` only; no hint families, with the candidate predicates noted in ADR-I-0024. +1. Five-column read contract; keep `embedding_text` only; no hint families, with the candidate predicates noted in ADR-I-0028. 2. Keep both text columns. 3. Drop both text columns. 4. Keep the unread hints for the planned phases. @@ -76,18 +76,18 @@ It stores what is read, keeps the one column that ceases to be re-derivable when Option 2: `content_text` is a deterministic function of graph object fields at every surface builder, so it never carries information graph authority lacks, and its one reader has its own ingest records; rejected outright. Option 3: `embedding_text` is cheap and is the only record of what a generated or caller-supplied vector embeds; rejected outright. -Option 4: no planned phase names a vector-layer predicate the existing fields could serve without new synchronisation work — scoped retrieval needs a scope id kept in sync by linking, temporal validity is a ranking property of new claim objects, salience evolves by reinforcement, and lifecycle hints describe vectors the write path deletes; reopened only under ADR-I-0024's prefilter rule. +Option 4: no planned phase names a vector-layer predicate the existing fields could serve without new synchronisation work — scoped retrieval needs a scope id kept in sync by linking, temporal validity is a ranking property of new claim objects, salience evolves by reinforcement, and lifecycle hints describe vectors the write path deletes; reopened only under ADR-I-0028's prefilter rule. Option 5 is the only subset with a forward-looking case that survives the synchronisation test; it was declined because no phase document asks for the predicate and the columns backfill cheaply when one does. ## Consequences - Positive: both adapters mirror one five-field manifest; the embedded shard stores those fields as payload beside the vector with keyword indexes on object id (the delete selector) and object type (the scope predicate); ADR-I-0023 owns the physical layout. -- Positive: the embedded surface is preserved as vector provenance, which matters once surfaces are generated rather than derived. -- Negative / tradeoffs: a scoped or time-bounded prefilter requires a backfill and a schema-version step rather than a query-only change; the candidate predicates noted in ADR-I-0024 make that step predictable. +- Positive: the embedded surface is preserved as vector provenance, which matters when surfaces are generated rather than derived. +- Negative / tradeoffs: a scoped or time-bounded prefilter requires a backfill and a schema-version step rather than a query-only change; the candidate predicates noted in ADR-I-0028 make that step predictable. ## Decision Boundary -Invariant: the vector record carries only fields a reader consumes, plus the embedded surface as provenance and the schema version ADR-I-0007 requires; readable content is hydrated from graph authority by object id; a returning hint arrives with its predicate and parity fixture under ADR-I-0024's prefilter rule. +Invariant: the vector record carries only fields a reader consumes, plus the embedded surface as provenance and the schema version ADR-I-0007 requires; readable content is hydrated from graph authority by object id; a returning hint arrives with its predicate and parity fixture under ADR-I-0028's prefilter rule. Not covered: the physical encoding of each column per adapter, and graph authority's own denormalised fields. @@ -99,8 +99,8 @@ Not covered: the physical encoding of each column per adapter, and graph authori ## Revisit When -- A retrieval route needs a scoped or time-bounded semantic search — add the column under ADR-I-0024's prefilter rule; this record's invariant is satisfied by a column that arrives with its reader. -- The assisted-remember phase makes the embedding surface a graph-authoritative provenance artifact — the vector copy becomes a cache and this record's provenance argument moves to the graph. +- A retrieval route needs a scoped or time-bounded semantic search — add the column under ADR-I-0028's prefilter rule; this record's invariant is satisfied by a column that arrives with its reader. +- The assisted-remember phase makes the embedding surface a graph-authoritative provenance artifact — the vector copy becomes a cache and graph authority owns the provenance argument under that decision. - A re-indexing workflow appears that cannot rebuild from graph authority — the readable-text question reopens with that workflow as its reader. ## Consultation impact @@ -111,4 +111,4 @@ Question asked: whether the unread hint families and the readable text column sh - ADR-I-0005 remains authoritative for graph authority over relationships; this record supersedes its payload field list and its "payload metadata as candidate filter" implementation guidance. - ADR-I-0002 remains authoritative for natural-language embedding surfaces; this record supersedes only its note to persist both text columns. -- ADR-I-0024 (completeness verdict and prefilter rule), ADR-I-0023 (embedded shard layout), ADR-I-0026 (evaluation baseline reader). +- ADR-I-0024 (completeness verdict), ADR-I-0028 (prefilter admission), ADR-I-0023 (embedded shard layout), ADR-I-0026 (evaluation baseline reader). diff --git a/docs/decisions/implementation/ADR-I-0026-raw-vector-baselines-read-the-retrieval-trace.md b/docs/decisions/implementation/ADR-I-0026-raw-vector-baselines-read-the-retrieval-trace.md index cedd0578..48dd2b5d 100644 --- a/docs/decisions/implementation/ADR-I-0026-raw-vector-baselines-read-the-retrieval-trace.md +++ b/docs/decisions/implementation/ADR-I-0026-raw-vector-baselines-read-the-retrieval-trace.md @@ -8,7 +8,7 @@ informed: [] warrant: warranted_by: "without this record, future work would likely add a public raw vector search to the facade for the evaluation baseline, or let evaluation tooling read a store's physical schema directly again, because each is the shortest path to a number" detected_signals: "externally observable contract shape with a tempting alternative; rejected alternative likely to be re-proposed; cross-repository obligation; deliberately bounded scope (no product use case for raw recall exists)" - cost_of_violation: "a raw-recall facade makes the library a vector-database abstraction and exposes unverified candidates as if they were memory; a schema-reading baseline breaks silently the moment a second vector adapter ships a different physical schema, and it reimplements canonical ordering the library already owns" + cost_of_violation: "a raw-recall facade makes the library a vector-database abstraction and exposes unverified candidates as if they were memory; a schema-reading baseline breaks silently the moment a second vector adapter ships a different physical schema, and it reimplements canonical ordering owned by the library" cost_of_wrong_preservation: "if a product use case for candidate-level recall arrives and this record is preserved as a blanket prohibition, the diagnostic surface the observability phase plans would be blocked instead of designed" cost_of_over_extension: "reading this record as forbidding evaluation tooling from using the trace at all would leave the baseline with no honest data source" depends_on: [implementation/ADR-I-0020-restart-identity-via-caller-supplied-ids-not-a-lookup-surface.md, implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md] diff --git a/docs/decisions/implementation/ADR-I-0027-embedded-vector-engine-runs-on-a-blocking-owner-that-flushes-every-write.md b/docs/decisions/implementation/ADR-I-0027-embedded-vector-engine-runs-on-a-blocking-owner-that-flushes-every-write.md index ea7c8dbd..bf50e7f5 100644 --- a/docs/decisions/implementation/ADR-I-0027-embedded-vector-engine-runs-on-a-blocking-owner-that-flushes-every-write.md +++ b/docs/decisions/implementation/ADR-I-0027-embedded-vector-engine-runs-on-a-blocking-owner-that-flushes-every-write.md @@ -6,7 +6,7 @@ deciders: ["ebigunso"] consulted: ["Claude Fable 5.1"] informed: [] warrant: - warranted_by: "without this record, future work would likely call the embedded engine directly from the async retrieval path (it is a plain synchronous API and the first implementation compiles), and would rely on the shard's final drop for persistence because the engine's write call returns success before anything is durable; both were the natural first draft of this phase" + warranted_by: "without this record, future work would likely call the embedded engine directly from the async retrieval path (it is a plain synchronous API and the first implementation compiles), and would rely on the shard's final drop for persistence because the engine's write call returns success before anything is durable; both appeared in the implementation draft reviewed on 2026-09-03" detected_signals: "cross-boundary contract shape (an engine with synchronous, non-durable writes inside an async host); premises likely to expire (the engine is beta and its persistence model may change); costly to detect (lost writes surface as a character forgetting after a crash, long after the cause)" cost_of_violation: "an engine call on an executor thread stalls every other retrieval in the process for the duration of a scan, build, or flush; a write acknowledged before its flush is lost on any exit that skips the shard's drop, and the loss is silent — the store reopens cleanly and simply lacks the memories" cost_of_wrong_preservation: "if the engine starts replaying its log on load or persisting on write and this record is preserved, every write keeps paying a synchronous disk sync it no longer needs" @@ -89,7 +89,7 @@ Not covered: the channel and thread mechanics, the backoff bound, and the batchi - The engine persists on write or replays its log on load (the canary fails in that direction) — the per-write flush becomes optional and this record is revised. - The write-burst measurement shows the per-write flush dominating ingestion cost — batch flushes behind an explicit acknowledgement rather than weakening the durability rule. -- The engine's directory lock changes semantics (the canary fails in that direction) — the constructor's wait-for-release rule is re-derived before the pin moves. +- The engine's directory lock changes semantics (the canary fails in that direction) — the constructor's wait-for-release rule is re-derived before adopting a different engine pin. - A multi-process deployment shape is designed — the single-owner lock discipline is reconsidered with the graph and statistics stores, never alone. ## Consultation impact diff --git a/docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md b/docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md new file mode 100644 index 00000000..9f25f796 --- /dev/null +++ b/docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md @@ -0,0 +1,101 @@ +--- +status: accepted +adr_type: implementation +date: 2026-09-04 +deciders: ["ebigunso"] +consulted: ["Claude Fable 5.1"] +informed: [] +warrant: + warranted_by: "without this record, future work would likely restore a three-valued vector-layer hint predicate that matches unknown values because that shape appears to preserve recall, repeating the pre-ADR-I-0028 behavior" + detected_signals: "cross-boundary contract shape (prefilter admission across two adapters); rejected alternative likely to be re-proposed; meaningful backfill and synchronisation cost; scope boundary is deliberate because graph authority may reason over incomplete knowledge" + cost_of_violation: "a prefilter over missing or stale values silently excludes reachable memories or admits them under a false rationale, producing continuity loss and misleading retrieval evidence that are expensive to diagnose" + cost_of_wrong_preservation: "if a stored predicate column is proven fully populated and cannot become unknown, preserving a special unknown-handling branch adds unreachable policy to both adapters" + cost_of_over_extension: "applying this vector-candidate admission rule to graph-authority reasoning would confuse incomplete domain knowledge with a corrupt denormalized index column" +depends_on: [implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md, implementation/ADR-I-0025-vector-record-is-a-read-contract.md] +implements: [] +supersedes: [] +superseded_by: null +supersession_scope: null +--- + +# ADR-I-0028: A vector-layer prefilter reads only a fully populated, immutable or synchronised column, and an unknown value never matches + +## Context and Problem Statement + +The vector candidate port previously carried currentness predicates implemented as match-or-unknown: a record whose payload lacked the field satisfied a positive predicate in both the service adapter and the test fake, so the filter admitted records under a rationale that was not true of them. +Those fields were written only at upsert. A value changed in graph authority could therefore remain stale in the vector payload, causing the prefilter to admit a record that should be excluded or silently exclude a memory that should remain reachable. +The speculative filters were deleted when no caller used them. The five-field read contract in ADR-I-0025 deliberately leaves a re-entry path for a justified predicate, so both adapters need one durable admission rule for any additional prefilter column. + +## Decision Drivers + +- A prefilter false negative is a memory that silently never returns, while a false positive consumes candidate capacity and presents a rationale that is not true. +- A denormalized predicate column is trustworthy only if every searchable record has a value and every mutation preserves it, or if the value cannot change. +- Both vector adapters implement the same port and must apply the same predicate semantics. +- Graph authority verifies candidates and remains the correct layer for questions that cannot be represented by a complete, current vector column. + +## Decision + +A vector-layer predicate may read a column only after the column is fully populated for every searchable record, including any required backfill from graph authority before the predicate is enabled, and only when the value is immutable or synchronised on every mutation. +Under those conditions a missing or unknown value is a defect, not an admissible state, and it never satisfies a positive predicate. This rule produces no false negative on a correctly populated column and turns an incorrectly populated one into a visible failure rather than silent widening. +A predicate that needs a value the write paths do not keep current, or that is not populated for every searchable record, remains a graph-authority question and is not a vector-layer prefilter. +Any admitted predicate lands in both adapters with a parity fixture. + +## Character Memory Relevance + +A character that cannot retrieve an episode because a stale hint excluded it appears to have never lived it, while a blank value admitted as a match gives a false explanation for recall. +Requiring complete and current predicate columns keeps candidate recall from becoming the hidden reason a memory is unreachable and keeps every stated filter rationale true. + +## Implementation Impact + +- The v0.1.6 object-type scope satisfies the rule because every vector record carries its object type and that identity field does not mutate. +- An additional vector predicate owns its column, graph-authority backfill, mutation synchronisation when applicable, schema step, both adapter mappings, and parity evidence as one change. +- ADR-I-0025 remains the record contract; a returning column earns its place through a concrete predicate and reader rather than speculative storage. + +## Considered Options + +1. Require complete population plus immutability or synchronisation, and never match unknown values. +2. Match unknown values to avoid false negatives. +3. Reject unknown values but allow predicate columns without a backfill prerequisite. +4. Keep graph-derived hints in every vector record in anticipation of possible predicates. + +## Decision Outcome + +Chosen option: **Option 1**. +It makes a prefilter's rationale true at the storage boundary, prevents silent exclusion from incomplete backfills, and gives both adapters one testable admission contract. + +### Rejected Alternatives + +Option 2 widens results under a rationale the stored value does not establish and hides incomplete population; rejected outright. +Option 3 permits pre-existing searchable records to disappear from results as soon as the predicate is enabled; rejected outright. +Option 4 mirrors unread, stale state across two adapters without a consumer and repeats the payload design that ADR-I-0025 replaced; rejected outright. + +## Consequences + +- Positive: a vector prefilter cannot silently rely on missing or stale denormalized state. +- Positive: the column, its reader, and the work that keeps it trustworthy enter together. +- Negative / tradeoffs: a scoped or time-bounded semantic search must pay the backfill and synchronisation or immutability cost before it can filter in the vector layer. + +## Decision Boundary + +Invariant: a vector-layer prefilter reads only a fully populated column that is immutable or synchronised on every mutation, and an unknown value never satisfies a positive predicate. + +Not covered: graph-authority query semantics, the v0.1.6 object-type scope shape, or the physical encoding of an admitted column in either adapter. + +## Validation + +- A census of both vector adapters shows no match-or-unknown condition. +- Any admitted predicate has a pre-enablement backfill or proof of complete population, a mutation-consistency design, and a parity fixture across both adapters. + +## Revisit When + +- Every value a predicate can read is structurally guaranteed present and cannot become unknown — the explicit unknown arm may be removed as unreachable while the complete-population rule remains. +- A vector backend cannot expose missing values distinctly enough to enforce the rule — that backend's admission evidence is re-derived before the predicate is implemented. + +## Consultation impact + +Question asked: whether the deleted hint filters should return for the embedded adapter; the ruling adopted this admission rule instead. The prefilter decision was separated from ADR-I-0024 on 2026-09-04 so each record has one governing claim. + +## More Information + +- ADR-I-0024 governs vector recall completeness and telemetry; ADR-I-0025 governs the five-field vector record. +- Candidate predicates that satisfy this rule, noted for whichever version needs them and binding on none: a scope id written at upsert and kept in sync by the link and reflection write paths; an immutable time window over `created_at` and `observed_at`, backfilled from graph authority before enablement. diff --git a/docs/design/database/vector_payload_design.md b/docs/design/database/vector_payload_design.md index de566b45..cf829355 100644 --- a/docs/design/database/vector_payload_design.md +++ b/docs/design/database/vector_payload_design.md @@ -1,6 +1,6 @@ # Vector Database Payload Design -> Current contract: [ADR-I-0025](../../decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md) supersedes the former denormalized payload-hint inventory with the five-field read contract documented here. [ADR-I-0024](../../decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md) governs any future prefilter re-entry. +> Current contract: [ADR-I-0025](../../decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md) supersedes the former denormalized payload-hint inventory with the five-field read contract documented here. [ADR-I-0028](../../decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md) governs any future prefilter re-entry. This document describes the Qdrant record contract for Character Memory. Qdrant is the semantic candidate index, while Oxigraph is the authority for memory content, relationships, provenance, lifecycle state, and currentness. @@ -70,7 +70,7 @@ Graph writes may succeed while vector maintenance fails. Public outcomes therefo The five-field change does not bump schema_version. Existing points may still contain obsolete extra fields; readers ignore those fields, and new writes emit only the five-field contract. No in-place payload migration is required. A rebuild from graph authority removes old extras naturally. -A future change that alters the meaning or required interpretation of the five fields must use the repository's schema-version policy. Adding graph-derived prefilter columns also requires an explicit consistency design: every write path must synchronize them, or the values must be immutable, and unknown values must never match. +A future change that alters the meaning or required interpretation of the five fields must use the repository's schema-version policy. A graph-derived prefilter column must be fully populated for every searchable record, including any required backfill before enablement, and must either be immutable or be synchronised by every write path; unknown values never match. ## Indexing Admission diff --git a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md index 5f7f1fc8..b306edee 100644 --- a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md +++ b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md @@ -1,6 +1,6 @@ # v0.1.6 Design: Embedded Vector Candidate Recall -Status: finished 2026-09-04 (ADR-I-0023 through ADR-I-0027); supersedes the 2026-07 draft of this document. The embedded engine ruling (Qdrant Edge over an in-house scan) was taken on two feasibility spikes; companion-repository evaluation evidence that was not yet produced at closeout remains explicitly pending in that repository. +Status: finished 2026-09-04 (ADR-I-0023 through ADR-I-0028); supersedes the 2026-07 draft of this document. The embedded engine ruling (Qdrant Edge over an in-house scan) was taken on two feasibility spikes; companion-repository evaluation evidence that was not yet produced at closeout remains explicitly pending in that repository. ## Version intent @@ -21,12 +21,12 @@ The embedded engine is the same family as the service backend, so payload and fi ## Design direction -### The port contract (ADR-I-0024, ADR-I-0025) +### The port contract (ADR-I-0024, ADR-I-0025, ADR-I-0028) This phase fixes the port contract deliberately, because two adapters cannot be held to an implicit one. Query: the embedding, the limit, and an object-type scope, and nothing else; an empty scope selects zero candidates, and the retrieval context rejects an empty configured object-type set at the boundary. -A vector-layer predicate reads only immutable or synchronised stored values, and an unknown or missing value never satisfies a positive predicate (ADR-I-0024); a future predicate lands in both adapters with a parity fixture, as every port change does. +A vector-layer predicate is enabled only after its stored column is fully populated for every searchable record, including any required backfill, and only when the value is immutable or synchronised on every mutation; an unknown or missing value never satisfies a positive predicate (ADR-I-0028). A future predicate lands in both adapters with a parity fixture, as every port change does. Result: a completeness envelope, the canonical candidates plus a typed verdict — not requested (the limit was zero or the scope was empty, so no search was issued), exhaustive (every scoped record was scored, so the requested top-K is determinate), boundary tie closed (an index returned a prefix and the cutoff cohort within that prefix was verified closed), or boundary tie open (the overfetch bound was reached with the cohort within the prefix still open). The boundary verdicts describe the index's answer, not global recall: an approximate index may omit records it never surfaced, so only the exhaustive verdict asserts determinate membership over the scoped population, while the boundary verdicts assert that the returned set is deterministic for a given index state and whether its cutoff cohort was closed. @@ -52,7 +52,7 @@ This phase ships the threshold at its exact-scan setting (the spike confirmed th Shard: cosine distance at the configured vector size; the five-field payload with keyword indexes on object id (the delete selector removes every surface of an object by it) and on object type (the scope predicate); the object-type scope expressed as a filter; the general shard type only (the update-only shard type carries an unimplemented path); object payloads only (the engine's point constructor panics on any other JSON shape). Search: the query runs through the service adapter's tie-closure loop and the canonical constructor, because the spike found identical-vector cohorts stable within a shard and across reopen but not across freshly built shards — deterministic admission comes from closing the cutoff cohort and ordering it canonically, never from engine order. -Verdict mapping: exhaustive when the shard is unindexed (known from the adapter's own threshold configuration) and the shared loop closed the cutoff cohort, with the scanned count taken from a filtered count of the scope rather than from the rows returned, so every exact scan with a closed cutoff reports exhaustive regardless of population size; boundary tie closed when an indexed shard's cutoff cohort closed (a statement about the index's returned prefix, not about global recall); boundary tie open whenever the bound is reached with the cohort still open, on an exact scan too; not requested at limit zero or empty scope. +Verdict mapping: exhaustive when the shard is unindexed (known from the adapter's own threshold configuration) and the shared loop closed the cutoff cohort, with `scanned` equal to the number of scoped records actually scored and never a truncated prefix, so every exact scan with a closed cutoff reports exhaustive regardless of population size; boundary tie closed when an indexed shard's cutoff cohort closed (a statement about the index's returned prefix, not about global recall); boundary tie open whenever the bound is reached with the cohort still open, on an exact scan too; not requested at limit zero or empty scope. Delete: remove every surface of each object id, matching the service adapter's selector. Restart safety: opening an existing shard validates its recorded vector size and distance against the configured embedding model and the adapter-owned marker's record schema version against the supported version before any query, raising the collection-compatibility error or the clear unsupported-schema failure ADR-I-0007 requires. Blocking discipline and durability (ADR-I-0027): every engine call — shard open and load with the lock backoff, payload index creation, upsert and delete, search, the filtered scope count behind the exhaustive verdict, index build, and the final drop — runs on a dedicated blocking owner the adapter creates at construction and that opens the shard itself, so the async composition entry point never touches the engine; the owner acknowledges a write only after the engine's flush, because the pinned engine persists only on flush and does not replay its log on load (writers that skipped the drop reopened with zero of two hundred points); the facade drop only signals the owner and no port or facade method is added. @@ -122,7 +122,7 @@ named-vector coexistence of two embedding spaces (an engine capability this deci migration tooling between modes or between record shapes (rebuild-from-graph-authority is the documented path) multi-process access to the embedded store (same single-process expectation as embedded graph storage) synchronisation between an embedded shard and a service collection (an engine capability; not exercised this phase) -any vector-layer predicate beyond the object-type scope (the candidate predicates noted in ADR-I-0024 belong to later phases) +any vector-layer predicate beyond the object-type scope (the candidate predicates noted in ADR-I-0028 belong to subsequent phases) any new public facade method (the evaluation baseline consumes the retrieval trace) reconciliation diagnostics (the reconciliation slice was deleted in the structured-verdict phase; graph verification is the guard) ``` @@ -188,7 +188,7 @@ Each item was parked on this phase by the structured-verdict phase; each row sta 4. Hint filter semantics. Parked claim: query-side hint semantics belong to the port contract. Re-verified: the filter type and both match-or-unknown implementations were deleted in the structured-verdict phase, and no consumer asks for a vector-layer predicate (the evaluation surface policy carries object types and budgets only). - Evidence: zero-hit census for the filter type and for empty-or-null match conditions in the service adapter; the prefilter rule and candidate predicates are recorded in ADR-I-0024. + Evidence: zero-hit census for the filter type and for empty-or-null match conditions in the service adapter; the prefilter rule and candidate predicates are recorded in ADR-I-0028. 5. Evaluation baseline capability. Parked claim: the baseline re-implements a hidden raw-vector capability against the payload schema. Re-verified: one singleton-scoped traced retrieval per measured kind reproduces the direct per-kind search exactly, which a sliced mixed-kind top-K would not; each retrieval's completeness verdict reports whether that kind's top-K was determinate over the scoped population (exhaustive) or only closed within the index's returned prefix (boundary verdicts), and the baseline records which; the evaluation adapter can hold item text from ingest. @@ -204,6 +204,6 @@ Each item was parked on this phase by the structured-verdict phase; each row sta - Dependency weight: the unstripped delta is recorded; the stripped delta and feature trimming are measured in-phase, and a material change reopens ADR-I-0023. - Parity suite placement: contract parity in the library, behaviour parity in the evaluation repository (above). - Settings shape: separate mode and path keys with `collection_name` as the backend-neutral namespace key naming one shard directory per collection, not a connection string interpreted by mode (ADR-I-0023). -- Hint families: all dropped from the vector record, with the two candidate predicates noted (ADR-I-0024, ADR-I-0025). +- Hint families: all dropped from the vector record, with the two candidate predicates noted (ADR-I-0025, ADR-I-0028). - Text columns: readable text dropped, embedded text kept as provenance, governed by the three sentences in ADR-I-0025. - Evaluation baseline: trace-sourced, no facade change (ADR-I-0026). diff --git a/docs/roadmap/development_roadmap.md b/docs/roadmap/development_roadmap.md index d00fccfe..4c6b874c 100644 --- a/docs/roadmap/development_roadmap.md +++ b/docs/roadmap/development_roadmap.md @@ -259,7 +259,7 @@ Assisted remember workflows may accept raw or semi-raw input as transient proces | v0.1.3 | Remember intake interfaces and deterministic write planning | Finished. Generation-ready write path with `RememberWritePlan`, memory candidates, validation, deterministic helpers, prepare/validate/commit flow, and shared manual/future-generated commit machinery. | | v0.1.4 | Continuity evaluation harness | Finished. Deterministic long-horizon evaluation harness implemented in the public companion `CharacterMemoryEvals` repository as a development aid, not core library functionality: synthetic interaction fixtures, a minimal example assistant loop, continuity-oriented retrieval-quality metrics, selectivity/fanout measurement, and hub-entity stress scenarios. | | v0.1.5 | Eval-driven v0.1 family closeout | Finished. Ran the evaluation harness across the v0.1 family, dispositioned eleven findings (none critical, none open), fixed deterministic vector admission and write-path warning diagnostics in the library, retained the measured defaults with a recorded basis (ADR-I-0022), adopted embedded persistent Oxigraph as the validated default (ADR-I-0021), and expanded the evaluation suite to 33 scenarios including benchmark-adapted and real-embedding fixtures. Closeout report: [`v0_1_5_closeout_report.md`](v0_1_5_closeout_report.md). | -| v0.1.6 | Embedded vector candidate recall | Finished 2026-09-04. An embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) is the default vector mode at its exact-scan indexing threshold, so zero-infrastructure local deployments and the default test path need no external service; the service adapter remains the explicit service mode. The redesigned port reports recall completeness, accepts only object-type scope, and stores the five-field record shared by both adapters. Companion-repository evaluation work is tracked there. Decisions: ADR-I-0023 through ADR-I-0027. | +| v0.1.6 | Embedded vector candidate recall | Finished 2026-09-04. An embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) is the default vector mode at its exact-scan indexing threshold, so zero-infrastructure local deployments and the default test path need no external service; the service adapter remains the explicit service mode. The redesigned port reports recall completeness, accepts only object-type scope, and stores the five-field record shared by both adapters. Companion-repository evaluation work is tracked there. Decisions: ADR-I-0023 through ADR-I-0028. | | v0.2 | Scoped continuity and reflection | `ContinuityScope`, scoped reflection, relationship state between arbitrary entities, character signals for continuing entities, open-loop/commitment lifecycle, and current continuity views. | | v0.3 | Factual rigor, temporal validity, and entity evolution | Assertions, claims, evidence links, belief assessments, source assessment, temporal validity, entity drift handling, and current-belief views. | | v0.4 | Retrieval observability and governance | Retrieval traces, context subgraphs, validation rules, graph health reports, policy diagnostics, rejected expansion traces, cluster/activation diagnostics, and retention assessment. | @@ -1200,7 +1200,7 @@ v0.2 entry is explicitly confirmed against the closed v0.1 family. Detailed draft: [`v0_1_6_embedded_vector_candidate_recall.md`](../design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md) -Decisions: ADR-I-0023 (embedded Qdrant Edge is the default vector candidate store), ADR-I-0024 (vector candidate recall reports completeness and prefilters never match unknown values), ADR-I-0025 (the vector record is a read contract), ADR-I-0026 (raw vector baselines read the retrieval trace), ADR-I-0027 (the embedded engine runs on a blocking owner that flushes every write). +Decisions: ADR-I-0023 (embedded Qdrant Edge is the default vector candidate store), ADR-I-0024 (vector candidate recall reports completeness), ADR-I-0025 (the vector record is a read contract), ADR-I-0026 (raw vector baselines read the retrieval trace), ADR-I-0027 (the embedded engine runs on a blocking owner that flushes every write), ADR-I-0028 (vector prefilters require fully populated current columns and never match unknown values). ## Intent diff --git a/src/adapters/qdrant/store.rs b/src/adapters/qdrant/store.rs index 433221fb..e496ec5b 100644 --- a/src/adapters/qdrant/store.rs +++ b/src/adapters/qdrant/store.rs @@ -5,10 +5,10 @@ use std::{collections::HashMap, time::Duration}; use async_trait::async_trait; use qdrant_client::qdrant::{ - points_selector::PointsSelectorOneOf, vectors_config, Condition, CountPointsBuilder, - CreateCollectionBuilder, CreateFieldIndexCollectionBuilder, DeletePointsBuilder, Distance, - Filter, PointStruct, ScoredPoint, ScrollPointsBuilder, SearchPointsBuilder, - UpsertPointsBuilder, VectorParams, VectorsConfig, + points_selector::PointsSelectorOneOf, vectors_config, Condition, CreateCollectionBuilder, + CreateFieldIndexCollectionBuilder, DeletePointsBuilder, Distance, Filter, PointStruct, + ScoredPoint, ScrollPointsBuilder, SearchPointsBuilder, UpsertPointsBuilder, VectorParams, + VectorsConfig, }; use qdrant_client::{config::QdrantConfig, Qdrant, QdrantError}; @@ -187,36 +187,6 @@ impl QdrantVectorCandidateStore { .map(|point| qdrant_payload_to_match(&point.payload, 0.0)) .collect() } - - async fn scoped_count(&self, query: &VectorCandidateSearch) -> Result { - let request = CountPointsBuilder::new(&self.collection_name) - .filter(qdrant_candidate_filter(query)) - .exact(true) - .build(); - let count = self - .client - .count(request) - .await - .map_err(qdrant_error)? - .result - .ok_or_else(|| { - CustomError::VectorDatabaseError(VectorDatabaseError::new( - "qdrant", - VectorDatabaseErrorKind::Response, - None, - "Qdrant count response was missing result", - )) - })? - .count; - usize::try_from(count).map_err(|_| { - CustomError::VectorDatabaseError(VectorDatabaseError::new( - "qdrant", - VectorDatabaseErrorKind::Conversion, - None, - format!("Qdrant scope count {count} exceeds the platform maximum"), - )) - }) - } } fn validate_collection_vector_config( @@ -316,11 +286,6 @@ impl VectorCandidateStore for QdrantVectorCandidateStore { } else { usize::MAX }; - let scanned = if zero_norm { - Some(self.scoped_count(query).await?) - } else { - None - }; let closed = close_tie_cohort(query.limit, fetch_limit_cap, |fetch_limit| async move { if zero_norm { self.scroll_zero_norm_candidate_batch(query, fetch_limit) @@ -330,6 +295,7 @@ impl VectorCandidateStore for QdrantVectorCandidateStore { } }) .await?; + let scanned = zero_norm.then_some(closed.fetched); let completeness = closed.completeness(scanned); Ok(VectorCandidateRecall { candidates: closed.candidates, @@ -1182,7 +1148,7 @@ mod tests { #[tokio::test] #[ignore = "requires local Qdrant: docker compose -f docker-compose.qdrant.yml up -d and QDRANT_CONNECTION_STRING"] - async fn qdrant_candidate_store_live_scores_zero_norm_query_candidates_zero() { + async fn qdrant_candidate_store_live_reports_the_zero_norm_scored_scope() { let url = env::var("QDRANT_CONNECTION_STRING") .expect("QDRANT_CONNECTION_STRING is required for live Qdrant regression"); let collection_name = format!("cm_zero_norm_{}", Uuid::new_v4().simple()); @@ -1219,7 +1185,9 @@ mod tests { .all(|candidate| candidate.score == 0.0)); assert_eq!( recall.completeness, - VectorRecallCompleteness::Exhaustive { scanned: 2 } + VectorRecallCompleteness::Exhaustive { + scanned: recall.candidates.len(), + } ); let _ = store.client.delete_collection(&collection_name).await; } diff --git a/src/api/types/retrieval.rs b/src/api/types/retrieval.rs index 109c12f4..89712c38 100644 --- a/src/api/types/retrieval.rs +++ b/src/api/types/retrieval.rs @@ -282,13 +282,16 @@ pub struct RetrievalTelemetry { /// /// `NotRequested` means no vector search ran. `Exhaustive` means every record in /// the requested scope was scored through a path the adapter knows to be exhaustive, -/// the cutoff cohort closed, and `scanned` is that scope's population. This includes -/// an unindexed scan or a full-scope scroll such as the zero-norm path. +/// the cutoff cohort closed, and `scanned` is the number of scoped records that path +/// actually scored. This includes an unindexed scan or a full-scope scroll such as +/// the zero-norm path. /// `BoundaryTieClosed` means the shared fetch loop closed the cutoff score cohort -/// within an index-produced result prefix; `fetched` is the number of backend rows read. +/// within an index-produced result prefix; `fetched` is the size of the final prefix +/// read, not the cumulative rows read across the loop's growth steps. /// `BoundaryTieOpen` means the cohort was still open at `fetch_bound`; `fetched` -/// is the number of rows read. Indexed boundary verdicts are deterministic only -/// for the index-returned prefix and never claim population-level completeness. +/// is likewise the size of the final prefix read. Indexed boundary verdicts are +/// deterministic only for the index-returned prefix and never claim population-level +/// completeness. /// /// ``` /// use character_memory::api::types::VectorRecallCompleteness as ApiCompleteness; diff --git a/src/config/app_settings.rs b/src/config/app_settings.rs index 6f7b1a10..49c3fbd4 100644 --- a/src/config/app_settings.rs +++ b/src/config/app_settings.rs @@ -236,12 +236,15 @@ impl Settings { /// - `oxigraph_path`: Local filesystem path for the Oxigraph database /// - `openai_api_key`: API key for OpenAI services /// + /// The selected vector mode's location is retained here, then consumed and validated when the + /// `CharacterMemory` facade is constructed; this constructor does not validate that location. + /// /// # Returns /// /// A `Result` which is: /// /// - `Ok`: A new `Settings` instance with the provided configuration - /// - `Err`: A `CustomError` if any required settings are missing or invalid + /// - `Err`: A `CustomError` if the configuration cannot be parsed or its general settings are invalid pub fn new(config: Config) -> Result { let raw: RawSettings = config.try_deserialize().map_err(|e| { CustomError::ConfigParseError(format!("Failed to parse external configuration: {e}")) diff --git a/src/ports/vector_candidate.rs b/src/ports/vector_candidate.rs index 5ef20cd0..0cca498b 100644 --- a/src/ports/vector_candidate.rs +++ b/src/ports/vector_candidate.rs @@ -25,9 +25,9 @@ pub(crate) trait VectorCandidateStore: Send + Sync { /// The shared fetch loop closes every score tie that crosses the requested /// limit. `Exhaustive` is reported only when every record in the requested /// scope was scored through a path the adapter knows to be exhaustive and the - /// cutoff cohort closed; `scanned` is the scope count. This includes an unindexed - /// scan or a full-scope scroll. An index-produced result prefix reports whether - /// its boundary tie closed or remained open at the fetch bound. + /// cutoff cohort closed; `scanned` is the number of scoped records actually scored + /// by that path. An index-produced result prefix reports whether its boundary tie + /// closed or remained open at the fetch bound. async fn search_candidates( &self, query: &VectorCandidateSearch, From a20b7efff4a40e5178fff61860a9b8a976294e7e Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 16:42:05 +0900 Subject: [PATCH 07/14] =?UTF-8?q?=F0=9F=93=9D=20Six=20decision=20records:?= =?UTF-8?q?=20counts=20in=20the=20phase=20document=20and=20completed=20pla?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 2 +- .../roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index f112e9f7..0b246928 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -24,7 +24,7 @@ - As-built port: `src/ports/vector_candidate.rs`, `src/models/vector/candidate_record.rs`, `src/models/vector/record.rs`, `src/adapters/qdrant/{store,payload}.rs`, `src/policy/embedding_surface.rs`, `src/usecases/retrieve.rs`, `src/api/types/retrieval.rs`, `src/composition.rs`, `src/config/app_settings.rs`, `src/test_support.rs`. - Prerequisite in this repository, landed: the toolchain pin moved to the embedded engine's minimum (Rust 1.97.0) in its own change, merged 2026-09-02 as a88c117. - Prerequisite tracked in the evaluation repository: its evidence-integrity fixes must be merged before this phase cites any harness measurement. -- Repo reference docs consulted: the five ADRs; ADR-I-0018 (dependency direction; ports may import the public retrieval vocabulary under its named exception); ADR-I-0007 (schema versioning); ADR-I-0021 (embedded default pattern); rules in `docs/coding-agent/rules/`. +- Repo reference docs consulted: the six ADRs (ADR-I-0023 through ADR-I-0028); ADR-I-0018 (dependency direction; ports may import the public retrieval vocabulary under its named exception); ADR-I-0007 (schema versioning); ADR-I-0021 (embedded default pattern); rules in `docs/coding-agent/rules/`. ## Open Questions (max 3) - none (the draft's five open questions were ruled by the decider on 2026-09-02 and are recorded in the phase document and the ADRs). diff --git a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md index b306edee..3ffb3e31 100644 --- a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md +++ b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md @@ -98,7 +98,7 @@ restart-safety test for the embedded store; pipeline test over the embedded adap dependency-weight report: unstripped and stripped release deltas, effect of feature trimming latency benchmark: exhaustive scan across corpus sizes at the configured dimension; executor responsiveness under a concurrent scan documentation: settings, single-process expectation, threshold semantics, measured latency guidance, rebuild-from-graph-authority as the path between modes -five implementation ADRs (ADR-I-0023 through ADR-I-0027) with reciprocal partial-supersession frontmatter on ADR-I-0001, ADR-I-0002, ADR-I-0003, and ADR-I-0005 +six implementation ADRs (ADR-I-0023 through ADR-I-0028) with reciprocal partial-supersession frontmatter on ADR-I-0001, ADR-I-0002, ADR-I-0003, and ADR-I-0005 ``` Deletions that are deliverables, not side effects: From f4e2a7e60bc827aec61aa7e89b1969c69126b780 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 16:58:34 +0900 Subject: [PATCH 08/14] =?UTF-8?q?=F0=9F=93=9D=20Reciprocal=20partial=20sup?= =?UTF-8?q?ersession=20between=20ADR-I-0024=20and=20ADR-I-0028;=20the=20co?= =?UTF-8?q?mpleted=20plan=20records=20the=20cross-mode=20evidence=20waiver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 2 +- ...reports-completeness-and-prefilters-never-match-unknown.md | 4 ++-- ...fully-populated-current-columns-and-never-match-unknown.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 0b246928..0bfdca58 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -13,7 +13,7 @@ - Every row of the phase document's deferral-reconfirmation checklist has its evidence produced and cited in the Progress Log. - Every deletion listed under "Deletions that are deliverables" is gone, with a zero-hit census. - Both repositories' service-gated suites execute (not skip) under the service-backed CI job. -- One or more PRs per wave in this repository, stacked on the planning PR and merged by the decider; the evaluation repository's cross-mode comparison is available as consumed evidence at closeout. +- One or more PRs per wave in this repository, stacked on the planning PR and merged by the decider; the evaluation repository's cross-mode comparison is consumed as evidence when that repository produces it; by decider ruling (ADR-I-0023, 2026-09-03) it is a revisit trigger for the embedded default, not a closeout gate, so its pending state does not hold this plan open. ## Scope / Non-goals - Scope: the phase document's deliverables and deletions, all in this repository. diff --git a/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md b/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md index 71f162e9..a693ef24 100644 --- a/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md +++ b/docs/decisions/implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md @@ -14,8 +14,8 @@ warrant: depends_on: [implementation/ADR-I-0018-responsibility-boundary-modules-with-enforced-dependency-direction.md, implementation/ADR-I-0022-retain-measured-retrieval-defaults.md] implements: [] supersedes: [] -superseded_by: null -supersession_scope: null +superseded_by: implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md +supersession_scope: partial # the prefilter-admission clause moved to ADR-I-0028; this record stays authoritative for completeness reporting --- # ADR-I-0024: Vector candidate recall reports its completeness diff --git a/docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md b/docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md index 9f25f796..38f4993e 100644 --- a/docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md +++ b/docs/decisions/implementation/ADR-I-0028-vector-prefilters-require-fully-populated-current-columns-and-never-match-unknown.md @@ -13,9 +13,9 @@ warrant: cost_of_over_extension: "applying this vector-candidate admission rule to graph-authority reasoning would confuse incomplete domain knowledge with a corrupt denormalized index column" depends_on: [implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md, implementation/ADR-I-0025-vector-record-is-a-read-contract.md] implements: [] -supersedes: [] +supersedes: [implementation/ADR-I-0024-vector-candidate-recall-reports-completeness-and-prefilters-never-match-unknown.md] superseded_by: null -supersession_scope: null +supersession_scope: partial # takes over ADR-I-0024's prefilter-admission clause only; ADR-I-0024 remains authoritative for completeness reporting --- # ADR-I-0028: A vector-layer prefilter reads only a fully populated, immutable or synchronised column, and an unknown value never matches From a038cdcddcf1dfc86a6e23dce22c4a3ac66b6887 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 17:06:19 +0900 Subject: [PATCH 09/14] =?UTF-8?q?=F0=9F=93=9D=20Definition=20of=20Done:=20?= =?UTF-8?q?evaluation-repository=20rows=20are=20dispositioned=20as=20pendi?= =?UTF-8?q?ng=20there=20by=20ruling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 0bfdca58..29153cbb 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -10,7 +10,7 @@ ## Definition of Done - Every acceptance criterion in the phase document's "Acceptance criteria" section holds with recorded evidence. -- Every row of the phase document's deferral-reconfirmation checklist has its evidence produced and cited in the Progress Log. +- Every row of the phase document's deferral-reconfirmation checklist is dispositioned in the Progress Log: its evidence is produced and cited, or, for the rows the evaluation repository owns, it is recorded as pending there under the decider's ruling (the evaluation repository plans and tracks its own work; ADR-I-0023 records the cross-mode comparison as a revisit trigger, not a closeout gate). - Every deletion listed under "Deletions that are deliverables" is gone, with a zero-hit census. - Both repositories' service-gated suites execute (not skip) under the service-backed CI job. - One or more PRs per wave in this repository, stacked on the planning PR and merged by the decider; the evaluation repository's cross-mode comparison is consumed as evidence when that repository produces it; by decider ruling (ADR-I-0023, 2026-09-03) it is a revisit trigger for the embedded default, not a closeout gate, so its pending state does not hold this plan open. From 05a4f0285907f94293d40921849df3c1d954d412 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 17:09:22 +0900 Subject: [PATCH 10/14] =?UTF-8?q?=F0=9F=93=9D=20Goal=20and=20assumption=20?= =?UTF-8?q?A2=20distinguish=20produced=20evidence=20from=20companion=20evi?= =?UTF-8?q?dence=20consumed=20when=20produced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 29153cbb..ea3c18eb 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -6,7 +6,7 @@ - work_type: mixed ## Goal -- Deliver the phase described in `docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md` under ADR-I-0023 through ADR-I-0028: a redesigned vector port contract, a five-field vector record, an embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) as the default vector mode with the service adapter retained as the explicit service mode, and a shared contract suite over both adapters; the evaluation repository's move of its vector-only baseline onto the retrieval trace is planned there and consumed here as closeout evidence. +- Deliver the phase described in `docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md` under ADR-I-0023 through ADR-I-0028: a redesigned vector port contract, a five-field vector record, an embedded vector candidate store on the in-process build of the service backend (Qdrant Edge) as the default vector mode with the service adapter retained as the explicit service mode, and a shared contract suite over both adapters; the evaluation repository's move of its vector-only baseline onto the retrieval trace is planned there; its outputs are consumed here when that repository produces them and are not a closeout gate (decider ruling, ADR-I-0023). ## Definition of Done - Every acceptance criterion in the phase document's "Acceptance criteria" section holds with recorded evidence. @@ -31,7 +31,7 @@ ## Assumptions - A1: The embedded engine is `qdrant-edge` pinned exactly at 0.8.0 (beta); its API is guarded by a contract canary, and the pin is bumped only with a re-run of the canary and the parity suite. -- A2: The evaluation repository plans and tracks its own work; this plan consumes two of its outputs only: the trace-sourced baseline's A/B evidence (deferral-reconfirmation row 5) and the cross-mode comparison consumed as closeout evidence and as a revisit trigger for the embedded default. +- A2: The evaluation repository plans and tracks its own work; this plan consumes two of its outputs only, when that repository produces them: the trace-sourced baseline's A/B evidence (deferral-reconfirmation row 5) and the cross-mode comparison, which is a revisit trigger for the embedded default; neither is a closeout gate. ## Tasks From 33f045063ce253e41cb0abd01a76f21e90648595 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 17:19:05 +0900 Subject: [PATCH 11/14] =?UTF-8?q?=F0=9F=93=9D=20Every=20statement=20about?= =?UTF-8?q?=20companion=20evidence=20uses=20the=20when-produced,=20not-a-g?= =?UTF-8?q?ate=20rule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 4 ++-- .../ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md | 2 +- .../roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index ea3c18eb..0d7d39d8 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -120,7 +120,7 @@ Shrink the record and the typed manifest to the five fields; drop the hint carriers, the readable text column, the per-field index creation for dropped fields, the test-only field constants and the prose-assertion note constant; replace the service adapter's private enum token mappers and the pipeline's copy with one Display/FromStr per enum in the domain. Also own ADR-I-0024's zero-norm rule on the write side: the vector indexing service rejects a zero-norm record embedding as a typed per-record indexing failure (adding the error-vocabulary variant it needs) before any adapter sees it, with a unit test on the service and a parity-suite fixture that Task_4 inherits, so no adapter ever normalises a zero vector. Consolidating the surface enum to one domain definition touches its internal definition and re-export and the public copy, all owned here. Publish the maximum number of embedding surfaces per object kind as public policy next to the surface policy that defines it (ADR-I-0026), with a test that the published value matches the builders. Update the database documentation that advertises the old record: the schema cheat sheet, the database README, and the graph schema design note's cross-store section lose the hint fields, the graph URI, the lifecycle-hint drift diagnostics, and the readable text column, and point at the five-field contract. Schema version ruling: the stored schema version is retained, because every field the new contract reads is present in records written under the current version and the removal only drops fields no reader consumes; existing stored payloads with extra fields are tolerated unread. A version bump is required only if a later change adds a read field that older records lack (the re-entry paths in ADR-I-0024), and that change owns the bump and its backfill. - acceptance: - The manifest test asserts exactly five entries; both text-column producers except `embedding_text` are gone. - - Zero-hit census across both repositories for the dropped fields and for `content_text` readers (the evaluation repository removes its reader under its own plan; its zero-hit census is consumed as closeout evidence, not ordered here). + - Zero-hit census across both repositories for the dropped fields and for `content_text` readers (the evaluation repository removes its reader under its own plan; its zero-hit census is consumed when that repository produces it and is not a closeout gate, not ordered here). - One token mapping per enum; census shows no copy in adapters or use cases. - A zero-norm record embedding yields a typed per-record indexing failure and never reaches an adapter (ADR-I-0024); unit test present. - validation: @@ -237,7 +237,7 @@ Task identifiers 5 and 6 were evaluation-repository work and moved to that repository's own plan; identifiers are not reused. Each wave ends with reviewer approval and a PR stacked on the previous wave's PR (GitHub stack on the planning PR); the next wave branches from the approved tip without waiting for a merge, and the decider merges the entire stack in one go at phase end (Decision Log, 2026-09-03 and 2026-09-04). -Notification duty: any wave that changes a public vocabulary the evaluation repository converts exhaustively (the vector database error kinds in Wave 3, the telemetry field in Wave 1) is announced to that repository before merge; how and when that repository adopts the change is planned there, and this plan only consumes the resulting compatibility evidence at closeout. +Notification duty: any wave that changes a public vocabulary the evaluation repository converts exhaustively (the vector database error kinds in Wave 3, the telemetry field in Wave 1) is announced to that repository before merge; how and when that repository adopts the change is planned there, and this plan only consumes the resulting compatibility evidence when that repository produces it, not as a closeout gate. ## Rollback / Safety - Embedded mode is the default and the default-mode construction test asserts it; the service mode's behavior is unchanged except for the reported verdict, the shrunken record, and the intended empty-scope change (an empty object-type scope selects zero instead of searching unfiltered, and an empty configured scope is rejected at the boundary), all covered by the parity suite and the retrieval tests. diff --git a/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md b/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md index 99ddf0f6..86ce5a91 100644 --- a/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md +++ b/docs/decisions/implementation/ADR-I-0023-embedded-qdrant-edge-vector-candidate-store.md @@ -45,7 +45,7 @@ Add an embedded vector candidate store mode behind the existing vector candidate Embedded mode is the default vector mode. The service adapter remains fully supported as the service and cloud mode, selected explicitly; this decision adds a mode and deprecates nothing. The evidence that licenses the default is produced by the same change that introduces the mode: the shared parity suite proves both adapters identical below their indexing thresholds, including identical-vector tie cohorts, and the library's integration suite runs on the embedded adapter without a service, so the shipped default is the validated path (ADR-I-0021's rule). -The companion evaluation repository's cross-mode run is closeout evidence and a revisit trigger, not a gate: a difference between modes on its continuity suite reopens this record. +The companion evaluation repository's cross-mode run is a revisit trigger consumed when that repository produces it, not a gate: a difference between modes on its continuity suite reopens this record. The embedded store is single-process, matching the embedded graph store's expectation. Exactness is a threshold property, not a promise: below the configured indexing threshold a shard answers by exhaustive scan, above it the index answers, and in both cases the completeness verdict (ADR-I-0024) reports the boundary state of the returned top-K. diff --git a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md index 3ffb3e31..772294f3 100644 --- a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md +++ b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md @@ -161,13 +161,13 @@ No public facade change beyond the telemetry field and the published maximum-sur The companion evaluation repository is a development aid; its own work is planned and tracked there, and this document records only what its measurements let this phase decide. - The library exposes, through an ordinary traced retrieval, everything a raw-vector baseline needs: the vector candidates with scores and the completeness verdict in telemetry; the honest way to use them is one singleton-scoped traced retrieval per measured object kind with a limit of the section budget multiplied by the maximum surfaces per object, deduplicated by object. -- The cross-mode comparison (service mode against embedded mode on the continuity suite, identical baselines expected under the parity contract) is closeout evidence and a revisit trigger for the embedded default recorded in ADR-I-0023 (a difference between modes reopens it); it is consumed at the closeout task, not produced by this plan. +- The cross-mode comparison (service mode against embedded mode on the continuity suite, identical baselines expected under the parity contract) is a revisit trigger for the embedded default recorded in ADR-I-0023 (a difference between modes reopens it); it is consumed when the evaluation repository produces it and is not a closeout gate, and it is not produced by this plan. - No candidate-search facade or configuration surface is added for the evaluation repository; the one public addition made for its trace reading is the published maximum-surfaces-per-object-kind policy value (ADR-I-0026), and if its measurements ever require more, that is a library decision taken on its own record. ## Evaluation tie-in The evaluation repository is expected to run its continuity suite in both vector modes; identical scenario results are what the parity contract predicts, and a difference between modes is a revisit trigger for the embedded default (ADR-I-0023). -How that configuration is built and run is planned in the evaluation repository; this phase consumes the comparison at closeout and cites nothing else from it. +How that configuration is built and run is planned in the evaluation repository; this phase consumes the comparison when that repository produces it, not as a closeout gate, and cites nothing else from it. ## Deferral-reconfirmation checklist From cd22f7e00ea675cfbbf392ac93fe6a72b96be477 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 17:24:58 +0900 Subject: [PATCH 12/14] =?UTF-8?q?=F0=9F=93=9D=20Acceptance=20surfaces=20as?= =?UTF-8?q?sert=20only=20this=20repository's=20census;=20the=20evaluation?= =?UTF-8?q?=20repository's=20part=20is=20consumed=20when=20produced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 6 +++--- .../ADR-I-0025-vector-record-is-a-read-contract.md | 2 +- .../v0_1_6_embedded_vector_candidate_recall.md | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 0d7d39d8..5d2957c6 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -12,7 +12,7 @@ - Every acceptance criterion in the phase document's "Acceptance criteria" section holds with recorded evidence. - Every row of the phase document's deferral-reconfirmation checklist is dispositioned in the Progress Log: its evidence is produced and cited, or, for the rows the evaluation repository owns, it is recorded as pending there under the decider's ruling (the evaluation repository plans and tracks its own work; ADR-I-0023 records the cross-mode comparison as a revisit trigger, not a closeout gate). - Every deletion listed under "Deletions that are deliverables" is gone, with a zero-hit census. -- Both repositories' service-gated suites execute (not skip) under the service-backed CI job. +- This repository's service-gated suites execute (not skip) under the service-backed CI job; the evaluation repository's equivalent gate is its own plan's concern. - One or more PRs per wave in this repository, stacked on the planning PR and merged by the decider; the evaluation repository's cross-mode comparison is consumed as evidence when that repository produces it; by decider ruling (ADR-I-0023, 2026-09-03) it is a revisit trigger for the embedded default, not a closeout gate, so its pending state does not hold this plan open. ## Scope / Non-goals @@ -120,7 +120,7 @@ Shrink the record and the typed manifest to the five fields; drop the hint carriers, the readable text column, the per-field index creation for dropped fields, the test-only field constants and the prose-assertion note constant; replace the service adapter's private enum token mappers and the pipeline's copy with one Display/FromStr per enum in the domain. Also own ADR-I-0024's zero-norm rule on the write side: the vector indexing service rejects a zero-norm record embedding as a typed per-record indexing failure (adding the error-vocabulary variant it needs) before any adapter sees it, with a unit test on the service and a parity-suite fixture that Task_4 inherits, so no adapter ever normalises a zero vector. Consolidating the surface enum to one domain definition touches its internal definition and re-export and the public copy, all owned here. Publish the maximum number of embedding surfaces per object kind as public policy next to the surface policy that defines it (ADR-I-0026), with a test that the published value matches the builders. Update the database documentation that advertises the old record: the schema cheat sheet, the database README, and the graph schema design note's cross-store section lose the hint fields, the graph URI, the lifecycle-hint drift diagnostics, and the readable text column, and point at the five-field contract. Schema version ruling: the stored schema version is retained, because every field the new contract reads is present in records written under the current version and the removal only drops fields no reader consumes; existing stored payloads with extra fields are tolerated unread. A version bump is required only if a later change adds a read field that older records lack (the re-entry paths in ADR-I-0024), and that change owns the bump and its backfill. - acceptance: - The manifest test asserts exactly five entries; both text-column producers except `embedding_text` are gone. - - Zero-hit census across both repositories for the dropped fields and for `content_text` readers (the evaluation repository removes its reader under its own plan; its zero-hit census is consumed when that repository produces it and is not a closeout gate, not ordered here). + - Zero-hit census in this repository for the dropped fields and for `content_text` readers; the evaluation repository removes its reader under its own plan, and its census is consumed when that repository produces it and is not a closeout gate (not ordered here). - One token mapping per enum; census shows no copy in adapters or use cases. - A zero-norm record embedding yields a typed per-record indexing failure and never reaches an adapter (ADR-I-0024); unit test present. - validation: @@ -195,7 +195,7 @@ - Cargo.lock - depends_on: [Task_4] - description: | - Retire the deterministic vector fake and its embedding-bearing record type in favour of the embedded adapter opened on a temporary shard directory, as the phase document specifies, so tests exercise the persistence path (failure-injecting and recording fakes stay); collect the deferral-reconfirmation evidence for all five checklist rows; bump the package version to 0.1.6 in the manifest and lockfile as prior milestone closeouts did; mark the roadmap row finished; move the plan to completed. + Retire the deterministic vector fake and its embedding-bearing record type in favour of the embedded adapter opened on a temporary shard directory, as the phase document specifies, so tests exercise the persistence path (failure-injecting and recording fakes stay); disposition all five deferral-reconfirmation checklist rows (evidence for the rows this repository owns; a pending-there note for the rows the evaluation repository owns, consumed when produced); bump the package version to 0.1.6 in the manifest and lockfile as prior milestone closeouts did; mark the roadmap row finished; move the plan to completed. - acceptance: - Zero-hit census for the retired fake and record type. - All five checklist rows cite evidence in the Progress Log. diff --git a/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md b/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md index 764f1eb1..c0c239a6 100644 --- a/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md +++ b/docs/decisions/implementation/ADR-I-0025-vector-record-is-a-read-contract.md @@ -93,7 +93,7 @@ Not covered: the physical encoding of each column per adapter, and graph authori ## Validation -- A census of both repositories shows no reader of the dropped fields and no reader of `content_text`. +- A census of this repository shows no reader of the dropped fields and no reader of `content_text`; the evaluation repository's census is its own plan's evidence. - The manifest test asserts the five entries; the parity suite serialises and reads them through both adapters. - The evaluation baseline reproduces its results with text sourced from ingest records. diff --git a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md index 772294f3..8b6c8634 100644 --- a/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md +++ b/docs/design/roadmap-phases/v0_1_6_embedded_vector_candidate_recall.md @@ -151,7 +151,7 @@ A zero-norm record embedding is rejected at indexing as a typed per-record failu The engine contract canary passes on the pinned version. The dependency-weight report records unstripped and stripped release deltas and the effect of feature trimming. The default test path requires no vector service; service-gated suites continue to pass unchanged. -Both adapters persist exactly the five-field read contract; a census of both repositories shows no reader of a dropped field. +Both adapters persist exactly the five-field read contract; a census of this repository shows no reader of a dropped field; the evaluation repository retires its own reader under its plan, and its census is consumed when that repository produces it and is not a closeout gate. Documentation states the single-process expectation, the threshold semantics, the latency guidance, and the rebuild-from-authority path. No public facade change beyond the telemetry field and the published maximum-surfaces-per-object-kind policy value; no retrieval behaviour change in service mode for non-empty scopes and non-degenerate queries (an empty object-type scope now selects zero instead of searching unfiltered, and an empty configured scope is rejected at the boundary, both intended). ``` @@ -171,7 +171,7 @@ How that configuration is built and run is planned in the evaluation repository; ## Deferral-reconfirmation checklist -Each item was parked on this phase by the structured-verdict phase; each row states the parked claim, what was re-verified at design time, and the evidence the implementation must produce. +Each item was parked on this phase by the structured-verdict phase; each row states the parked claim, what was re-verified at design time, and the evidence the implementation must produce for the rows this repository owns, while rows owned by the evaluation repository are recorded as pending there and consumed when produced. 1. Canonical-candidates newtype survival. Parked claim: the newtype survives the port redesign or is absorbed into its result envelope. @@ -180,7 +180,7 @@ Each item was parked on this phase by the structured-verdict phase; each row sta 2. Dual text columns. Parked claim: the text columns' fate depends on the port's read contract. Re-verified: the readable text column had exactly one reader (the evaluation baseline) and the embedded text column none; the evaluation repository can source item text from its own ingest. - Evidence: zero-hit census for the readable text column across both repositories; a vector-only run before and after produces identical item identities and text. + Evidence: zero-hit census for the readable text column in this repository now; the evaluation repository's census and its before/after vector-only run (identical item identities and text) are consumed when that repository produces them and are not a closeout gate. 3. Search completeness. Parked claim: the port cannot express whether the top-K was determinate. Re-verified: the only degradation site is the service adapter's fetch bound; no pipeline path inspects or retries on it; the embedded engine's own tie order is not stable across fresh shards, which makes the shared loop a requirement for both adapters rather than a service-only workaround. From 25d131532d4638834b1429a1b2adf177bd7148ad Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 17:34:44 +0900 Subject: [PATCH 13/14] =?UTF-8?q?=F0=9F=93=9D=20Task=5F7=20acceptance:=20c?= =?UTF-8?q?hecklist=20rows=20are=20dispositioned,=20not=20all=20evidenced?= =?UTF-8?q?=20here?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 5d2957c6..88417d35 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -198,7 +198,7 @@ Retire the deterministic vector fake and its embedding-bearing record type in favour of the embedded adapter opened on a temporary shard directory, as the phase document specifies, so tests exercise the persistence path (failure-injecting and recording fakes stay); disposition all five deferral-reconfirmation checklist rows (evidence for the rows this repository owns; a pending-there note for the rows the evaluation repository owns, consumed when produced); bump the package version to 0.1.6 in the manifest and lockfile as prior milestone closeouts did; mark the roadmap row finished; move the plan to completed. - acceptance: - Zero-hit census for the retired fake and record type. - - All five checklist rows cite evidence in the Progress Log. + - All five checklist rows are dispositioned in the Progress Log: produced evidence for the rows this repository owns, an explicit pending-there note for the rows the evaluation repository owns. - Task_8 audit dispositions landed: one point-identity derivation (the v5 derivation) shared by both adapters with a parity assertion and no compat shim; one shared read function for the record contract returning the closed error vocabulary, with the service adapter's stringly database errors for payload and scroll-limit faults removed; the canary pins that an indexed shard searched exactly returns the exhaustive result; the service zero-norm verdict's `scanned` is the size of the final scroll response whose records were actually scored; the duplicate vector-config validation is reduced to one or justified in place; the stale header comments and README line are corrected; the public facade constructor and `Settings::new` rustdoc describe both vector-store modes in backend-neutral terms (embedded default with a store path; explicit service mode with a connection string). - The public completeness type documents all four verdict meanings and the `scanned` and `fetched` counters, stating that `scanned` is the number of scoped records actually scored and `fetched` is the final prefix size rather than a cumulative count, and that a closed boundary verdict is deterministic only for the index-returned prefix, never population-level completeness; the port trait's doc comment states the verdict guarantees (tie closure through the shared loop; exhaustive only when every scoped record was scored through a path the adapter knows to be exhaustive, including an unindexed scan or a full-scope scroll, with a closed cohort; open at the bound) so a future adapter cannot label an index-produced prefix exhaustive (deferred from the Task_2 review to avoid re-cascading the stack mid-wave). - validation: From 21d9786ec72d9d77f93128c7c2882f77702a7dc1 Mon Sep 17 00:00:00 2001 From: ebigunso Date: Fri, 4 Sep 2026 17:37:26 +0900 Subject: [PATCH 14/14] =?UTF-8?q?=F0=9F=93=9D=20Progress=20log:=20plan=20a?= =?UTF-8?q?pproval=20recorded=20(closes=20the=20pending=20note)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- .../plans/completed/v0-1-6-embedded-vector-recall-plan.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md index 88417d35..5a733ee0 100644 --- a/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md +++ b/docs/coding-agent/plans/completed/v0-1-6-embedded-vector-recall-plan.md @@ -250,6 +250,7 @@ Append-only editing rule (applies to both logs below): when appending an entry, - 2026-09-02 Planning wave completed: five parallel inputs (design consult, two altitude audits, two forensic censuses) consolidated; decider ruled the five design questions; ADR-I-0023 through ADR-I-0026, the rewritten phase document, and the roadmap section authored on branch `plan/v0-1-6-embedded-vector-recall`. Plan awaits approval. +- 2026-09-03 Plan approved for implementation by the decider (status approved in the first implementation change); the wave PRs are stacked on the planning PR and the stack merges in one go at phase end, so the 2026-09-02 note that plan approval was pending is closed by this entry. - 2026-09-04 Wave 1 done: Task_1 (a633b2e, PR #74) and Task_2 (5b30856, PR #75) approved by the Tier D reviewer; Task_2 needed one revision (checked backend-limit conversion with a BoundaryTieOpen regression at the u32 cap; public re-exports) and carries one post-review fix (dimension check before the zero-norm scroll). PRs stacked on the planning PR as stack #76; merges pending. Wave 2 (Task_3) dispatched on a branch stacked on Task_2. - 2026-09-04 Wave 2 done: Task_3 (36ef8c8, PR #77) approved by the Tier D reviewer after two revisions on the zero-norm rejected-record fixture (final shape: one definition behind the non-default `test-fixtures` feature, enabled for integration tests by a self dev-dependency, doc-hidden). The evaluation repository's exhaustive error-vocabulary conversion is recorded as its re-pin obligation (ADR-I-0023), not a finding. PR appended to stack #76. - 2026-09-04 — Merge shape refined: the stack (planning PR at the bottom, one PR per wave above it) is merged in one go at phase end; GitHub's bottom-up stack order blocks interim merges of wave PRs while the planning PR is a draft, which is the intended shape. Waves proceed by branching from the previous wave's approved tip.