Feature/tilemaxsim phase3

Cargo.toml

@@ -55,6 +55,9 @@ zerocopy.workspace = true
 [target.'cfg(all(any(target_arch = "x86_64", target_arch = "aarch64"), any(target_os = "linux", target_os = "macos")))'.dependencies]
 mimalloc = { version = "0.1.49", features = ["local_dynamic_tls"] }
 
+[target.'cfg(unix)'.dependencies]
+libc = "0.2.182"
+
 [lints]
 workspace = true
 

crates/vchordg/src/types.rs

@@ -56,7 +56,7 @@ impl VchordgIndexOptions {
         if !alpha.iter().all(|x| (1.0..2.0).contains(x)) {
             return Err(ValidationError::new("alpha is too large or too small"));
         }
-        if alpha[0] != 1.0 {
+        if alpha.first() != Some(&1.0) {
             return Err(ValidationError::new("`alpha` should contain `1.0`"));
         }
         Ok(())
@@ -144,3 +144,25 @@ impl<V> Structure<V> {
         self.children.is_empty()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::VchordgIndexOptions;
+
+    #[test]
+    fn validate_alpha_handles_empty_without_panicking() {
+        let empty: &[f32] = &[];
+        assert!(VchordgIndexOptions::validate_alpha(empty).is_err());
+    }
+
+    #[test]
+    fn validate_alpha_accepts_default() {
+        assert!(VchordgIndexOptions::validate_alpha(&[1.0, 1.2]).is_ok());
+    }
+
+    #[test]
+    fn validate_alpha_rejects_unsorted_or_out_of_range() {
+        assert!(VchordgIndexOptions::validate_alpha(&[1.2, 1.0]).is_err());
+        assert!(VchordgIndexOptions::validate_alpha(&[2.0]).is_err());
+    }
+}

crates/vchordrq/src/build.rs

@@ -119,6 +119,7 @@ pub fn build<R: RelationWrite, O: Operator>(
         height_of_root: structures.len() as u32,
         is_residual,
         rerank_in_heap: vchordrq_options.rerank_in_table,
+        indexed_vectors: Some(0),
         centroids_first: centroids.first(),
         vectors_first: vectors,
         centroid_prefetch: pointer_of_centroids

crates/vchordrq/src/bulkdelete.rs

@@ -25,7 +25,8 @@ pub fn bulkdelete<R: RelationRead + RelationWrite, O: Operator>(
     index: &R,
     check: impl Fn(),
     callback: impl Fn(NonZero<u64>) -> bool,
-) where
+) -> u64
+where
     R::Page: Page<Opaque = Opaque>,
 {
     let meta_guard = index.read(0);
@@ -38,6 +39,8 @@ pub fn bulkdelete<R: RelationRead + RelationWrite, O: Operator>(
 
     drop(meta_guard);
 
+    let mut live = 0_u64;
+
     let step = |state: State| {
         let mut results = Vec::new();
         for first in state {
@@ -64,19 +67,21 @@ pub fn bulkdelete<R: RelationRead + RelationWrite, O: Operator>(
             while current != u32::MAX {
                 check();
                 let read = index.read(current);
-                let flag = 'flag: {
+                let (flag, page_live) = 'scan: {
+                    let mut page_live = 0_u64;
                     for i in 1..=read.len() {
                         let bytes = read.get(i).expect("data corruption");
                         let tuple = FrozenTuple::deserialize_ref(bytes);
                         if let FrozenTupleReader::_0(tuple) = tuple {
                             for p in tuple.payload().iter() {
                                 if Some(true) == p.map(&callback) {
-                                    break 'flag true;
+                                    break 'scan (true, 0);
                                 }
+                                page_live += u64::from(p.is_some());
                             }
                         }
                     }
-                    false
+                    (false, page_live)
                 };
                 if flag {
                     drop(read);
@@ -89,9 +94,12 @@ pub fn bulkdelete<R: RelationRead + RelationWrite, O: Operator>(
                                 if Some(true) == p.map(&callback) {
                                     *p = None;
                                 }
+                                live += u64::from(p.is_some());
                             }
                         }
                     }
+                } else {
+                    live += page_live;
                 }
                 current = directory.next().unwrap_or(u32::MAX);
             }
@@ -101,16 +109,18 @@ pub fn bulkdelete<R: RelationRead + RelationWrite, O: Operator>(
             while current != u32::MAX {
                 check();
                 let read = index.read(current);
-                let flag = 'flag: {
+                let (flag, page_live) = 'scan: {
+                    let mut page_live = 0_u64;
                     for i in 1..=read.len() {
                         let bytes = read.get(i).expect("data corruption");
                         let tuple = AppendableTuple::deserialize_ref(bytes);
                         let p = tuple.payload();
                         if Some(true) == p.map(&callback) {
-                            break 'flag true;
+                            break 'scan (true, 0);
                         }
+                        page_live += u64::from(p.is_some());
                     }
-                    false
+                    (false, page_live)
                 };
                 if flag {
                     drop(read);
@@ -122,14 +132,18 @@ pub fn bulkdelete<R: RelationRead + RelationWrite, O: Operator>(
                         if Some(true) == p.map(&callback) {
                             *p = None;
                         }
+                        live += u64::from(p.is_some());
                     }
                     current = write.get_opaque().next;
                 } else {
+                    live += page_live;
                     current = read.get_opaque().next;
                 }
             }
         }
     }
+
+    live
 }
 
 pub fn bulkdelete_vectors<R: RelationRead + RelationWrite, O: Operator>(

crates/vchordrq/src/cost.rs

@@ -18,6 +18,7 @@ use index::relation::{Page, RelationRead};
 pub struct Cost {
     pub dim: u32,
     pub cells: Vec<u32>,
+    pub indexed_vectors: Option<u64>,
 }
 
 #[must_use]
@@ -27,8 +28,13 @@ pub fn cost<R: RelationRead>(index: &R) -> Cost {
     let meta_tuple = MetaTuple::deserialize_ref(meta_bytes);
     let dim = meta_tuple.dim();
     let cells = meta_tuple.cells().to_vec();
+    let indexed_vectors = meta_tuple.indexed_vectors();
 
     drop(meta_guard);
 
-    Cost { dim, cells }
+    Cost {
+        dim,
+        cells,
+        indexed_vectors,
+    }
 }

crates/vchordrq/src/lib.rs

@@ -24,9 +24,11 @@ mod freepages;
 mod insert;
 mod linked_vec;
 mod maintain;
+mod maxsim_cost;
 mod prewarm;
 mod rerank;
 mod search;
+mod statistics;
 mod tape;
 mod tape_writer;
 mod tuples;
@@ -43,9 +45,13 @@ pub use cost::cost;
 pub use fast_heap::FastHeap;
 pub use insert::{InsertChooser, insert, insert_vector};
 pub use maintain::{MaintainChooser, maintain};
+pub use maxsim_cost::{
+    MaxsimCostBackend, MaxsimCostEstimate, MaxsimCostInput, estimate_maxsim_cost,
+};
 pub use prewarm::prewarm;
 pub use rerank::{how, rerank_heap, rerank_index};
 pub use search::{default_search, maxsim_search};
+pub use statistics::set_indexed_vectors;
 
 use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};