From 3cd351535eb71a20f9a371c92aaa84963fdd5dd4 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 6 Sep 2026 20:03:08 +0800 Subject: [PATCH 1/3] perf: improve IVF-SQ training, encoding, and reader reuse --- core/src/index.rs | 8 +- core/src/ivfsq.rs | 119 +- core/src/ivfsq_io.rs | 429 ++++++- core/src/sq.rs | 407 +++++- core/src/topk.rs | 4 + docs/api.html | 13 +- docs/benchmarks/ivfsq-lance-20260906.json | 1372 +++++++++++++++++++++ docs/development.html | 29 +- docs/index.html | 91 +- docs/ivf-flat.html | 2 +- docs/ivf-rq.html | 4 +- docs/ivf-sq-performance.md | 208 ++++ docs/ivf-sq.html | 43 +- docs/releases.html | 2 + tools/README.md | 13 + tools/benchmark_ivfsq_reader.py | 105 ++ tools/benchmark_lance_ivfsq.py | 148 +++ 17 files changed, 2813 insertions(+), 184 deletions(-) create mode 100644 docs/benchmarks/ivfsq-lance-20260906.json create mode 100644 docs/ivf-sq-performance.md create mode 100644 tools/benchmark_ivfsq_reader.py create mode 100644 tools/benchmark_lance_ivfsq.py diff --git a/core/src/index.rs b/core/src/index.rs index 198eaed..5d3abca 100644 --- a/core/src/index.rs +++ b/core/src/index.rs @@ -1464,9 +1464,11 @@ impl VectorIndexReader { IVFFLAT_MAGIC => Ok(Self::IvfFlat(IVFFlatIndexReader::open_with_header( reader, header, )?)), - IVF_SQ_MAGIC => Ok(Self::IvfSq(IVFSQIndexReader::open_with_header( - reader, header, - )?)), + IVF_SQ_MAGIC => { + let mut reader = IVFSQIndexReader::open_with_header(reader, header)?; + reader.configure_cache(options.memory_budget_bytes); + Ok(Self::IvfSq(reader)) + } MAGIC => Ok(Self::IvfPq(IVFPQIndexReader::open_with_header( reader, header, )?)), diff --git a/core/src/ivfsq.rs b/core/src/ivfsq.rs index c39a62e..af69ed9 100644 --- a/core/src/ivfsq.rs +++ b/core/src/ivfsq.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -//! IVF with per-list, per-dimension 8-bit residual scalar quantization. +//! IVF with per-dimension 8-bit residual scalar quantization. use crate::coarse::CoarseAssignment; -use crate::distance::{fvec_madd, preprocess_vectors, MetricType}; +use crate::distance::{preprocess_vectors, MetricType}; use crate::ivfpq::RowIdFilter; use crate::kmeans::{self, KMeansConfig}; use crate::sq::ScalarQuantizer; @@ -86,9 +86,14 @@ impl IVFSQIndex { self.quantizer_centroids = kmeans::kmeans_train(&KMeansConfig::default(), &processed, n, self.d, self.nlist); self.coarse_assignment.reset(); - let (list_ids, residuals) = self.assign_residuals(&processed, n); - self.sq.train(&residuals, n); - self.train_list_sqs(&list_ids, &residuals); + let list_ids = self.coarse_assignment.assign( + &processed, + n, + &self.quantizer_centroids, + self.nlist, + self.d, + ); + self.train_list_sqs(&processed, &list_ids); } pub fn add(&mut self, data: &[f32], ids: &[i64], n: usize) { @@ -247,44 +252,39 @@ impl IVFSQIndex { } } - fn assign_residuals(&mut self, processed: &[f32], n: usize) -> (Vec, Vec) { - let list_ids = self.coarse_assignment.assign( - processed, - n, - &self.quantizer_centroids, - self.nlist, - self.d, - ); - let mut residuals = vec![0.0f32; n * self.d]; - for i in 0..n { - let vector = &processed[i * self.d..(i + 1) * self.d]; - self.write_residual( - vector, - list_ids[i], - &mut residuals[i * self.d..(i + 1) * self.d], - ); + fn train_list_sqs(&mut self, data: &[f32], list_ids: &[usize]) { + if list_ids.is_empty() { + self.sq = ScalarQuantizer::new(self.d); + self.list_sqs = vec![self.sq.clone(); self.nlist]; + return; } - (list_ids, residuals) - } - - fn train_list_sqs(&mut self, list_ids: &[usize], residuals: &[f32]) { - let mut list_residuals = vec![Vec::new(); self.nlist]; - for (i, &list_id) in list_ids.iter().enumerate() { - let residual = &residuals[i * self.d..(i + 1) * self.d]; - list_residuals[list_id].extend_from_slice(residual); + let mut list_rows = vec![Vec::new(); self.nlist]; + for (row, &list_id) in list_ids.iter().enumerate() { + list_rows[list_id].push(row); } - self.list_sqs = vec![self.sq.clone(); self.nlist]; - for (list_id, values) in list_residuals.iter().enumerate() { - if !values.is_empty() { - let mut sq = ScalarQuantizer::new(self.d); - sq.train(values, values.len() / self.d); - self.list_sqs[list_id] = sq; + let trained = list_rows + .par_iter() + .enumerate() + .map(|(list_id, rows)| { + (!rows.is_empty()).then(|| { + ScalarQuantizer::train_residual_rows(data, rows, self.list_centroid(list_id)) + }) + }) + .collect::>(); + let mut mins = vec![f32::INFINITY; self.d]; + let mut maxs = vec![f32::NEG_INFINITY; self.d]; + for sq in trained.iter().flatten() { + for dim in 0..self.d { + mins[dim] = mins[dim].min(sq.mins[dim]); + maxs[dim] = maxs[dim].max(sq.maxs[dim]); } } - } - - fn write_residual(&self, vector: &[f32], list_id: usize, out: &mut [f32]) { - fvec_madd(vector, self.list_centroid(list_id), -1.0, out); + self.sq = ScalarQuantizer::with_dimension_bounds(self.d, mins, maxs); + // A training sample may contain only a handful of rows in a partition. + // Its extrema severely clip unseen residuals (including constant sample + // dimensions). Pool the observed residual bounds across partitions; + // retain per-list metadata so existing v1 files keep their own bounds. + self.list_sqs = vec![self.sq.clone(); self.nlist]; } pub(crate) fn list_centroid(&self, list_id: usize) -> &[f32] { @@ -312,23 +312,46 @@ fn append_encoded_rows( output_codes: &mut Vec, ) { output_ids.reserve(rows.len()); - output_codes.reserve(rows.len().saturating_mul(d)); - let mut residual = vec![0.0f32; d]; - let mut code = vec![0u8; d]; - for &row in rows { - let vector = &data[row * d..(row + 1) * d]; - fvec_madd(vector, centroid, -1.0, &mut residual); - sq.encode(&residual, &mut code); - output_ids.push(input_ids[row]); - output_codes.extend_from_slice(&code); + if rows.is_empty() { + return; } + let start = output_codes.len(); + output_codes.resize(start + rows.len() * d, 0); + sq.encode_residual_rows(data, rows, centroid, &mut output_codes[start..]); + output_ids.extend(rows.iter().map(|&row| input_ids[row])); } #[cfg(test)] mod tests { use super::*; + use crate::distance::fvec_madd; use std::collections::HashSet; + #[test] + fn sparse_partition_bounds_do_not_collapse_unseen_residuals() { + let mut index = IVFSQIndex::new(2, 3, MetricType::L2); + index.set_quantizer_centroids(vec![0.0, 0.0, 10.0, 10.0, 20.0, 20.0]); + index.train_list_sqs(&[-2.0, -2.0, 10.0, 10.0, 12.0, 12.0], &[0, 1, 1]); + // Partition zero saw only one residual and partition two was empty. + // Neither should freeze future vectors at a sample's constant value. + index.add(&[0.0, 0.0, 20.0, 20.0], &[42, 43], 2); + let mut distances = [0.0; 2]; + let mut labels = [0; 2]; + index.search( + &[0.0, 0.0, 20.0, 20.0], + 2, + 1, + 3, + &mut distances, + &mut labels, + ); + assert_eq!(labels, [42, 43]); + assert!( + distances.iter().all(|&distance| distance < 0.001), + "{distances:?}" + ); + } + #[test] fn ivfsq_full_scan_recalls_added_vector() { let d = 4; diff --git a/core/src/ivfsq_io.rs b/core/src/ivfsq_io.rs index 6fb7c5b..5607a67 100644 --- a/core/src/ivfsq_io.rs +++ b/core/src/ivfsq_io.rs @@ -30,11 +30,14 @@ use crate::io::{ReadRequest, SeekRead, SeekWrite}; use crate::ivfpq::RowIdFilter; use crate::ivfsq::IVFSQIndex; use crate::kmeans; +use crate::read_options::VectorIndexReaderOptions; use crate::sq::ScalarQuantizer; use crate::topk::TopKHeap; use rayon::prelude::*; +use std::collections::VecDeque; use std::io; use std::mem::size_of; +use std::sync::Arc; pub const IVF_SQ_MAGIC: u32 = 0x49565351; // "IVSQ" pub const IVF_SQ_VERSION: u32 = 1; @@ -58,6 +61,7 @@ pub fn write_ivfsq_index(index: &IVFSQIndex, out: &mut dyn SeekWrite) -> io::Res }) })?; let sorted_lists = (0..index.nlist) + .into_par_iter() .map(|list_id| build_sorted_sq_list_metadata(index, list_id)) .collect::>>()?; @@ -121,20 +125,42 @@ pub fn write_ivfsq_index(index: &IVFSQIndex, out: &mut dyn SeekWrite) -> io::Res write_i32_le(out, list_counts[list_id])?; write_i32_le(out, list_id_bytes_lens[list_id])?; } - for (list_id, list) in sorted_lists.iter().enumerate() { - if list.order.is_empty() { - continue; + // Bound transposition scratch independently of the total index size. An + // oversized individual list still uses at most one list's code buffer. + const TRANSPOSE_BATCH_BYTES: usize = 16 * 1024 * 1024; + let mut start = 0; + while start < index.nlist { + let mut end = start; + let mut bytes = 0; + while end < index.nlist { + let next = index.codes[end].len(); + if end > start && next > TRANSPOSE_BATCH_BYTES.saturating_sub(bytes) { + break; + } + bytes += next; + end += 1; } - let codes = block_sorted_sq_codes( - &index.codes[list_id], - &list.order, - index.d, - IVF_SQ_SCAN_BLOCK_SIZE, - ); - out.write_all(&codes)?; - write_i64_le(out, list.base_id)?; - write_i32_le(out, usize_to_i32(list.id_bytes.len(), "delta ID section")?)?; - out.write_all(&list.id_bytes)?; + let blocked = (start..end) + .into_par_iter() + .map(|list_id| { + block_sorted_sq_codes( + &index.codes[list_id], + &sorted_lists[list_id].order, + index.d, + IVF_SQ_SCAN_BLOCK_SIZE, + ) + }) + .collect::>(); + for (list, codes) in sorted_lists[start..end].iter().zip(blocked) { + if list.order.is_empty() { + continue; + } + out.write_all(&codes)?; + write_i64_le(out, list.base_id)?; + write_i32_le(out, usize_to_i32(list.id_bytes.len(), "delta ID section")?)?; + out.write_all(&list.id_bytes)?; + } + start = end; } Ok(()) } @@ -152,9 +178,33 @@ pub struct IVFSQIndexReader { pub list_counts: Vec, pub list_id_bytes_lens: Vec, loaded: bool, + list_cache: Option, } impl IVFSQIndexReader { + /// Open with a bounded cache of decoded partitions. `open` retains the + /// uncached positional-I/O behavior for callers that manage their own cache. + pub fn open_with_options(reader: R, options: VectorIndexReaderOptions) -> io::Result { + let mut index = Self::open(reader)?; + index.configure_cache(options.memory_budget_bytes); + Ok(index) + } + + pub(crate) fn configure_cache(&mut self, memory_budget_bytes: usize) { + let resident = size_of::() + + self.quantizer_centroids.capacity() * size_of::() + + self.list_offsets.capacity() * size_of::() + + self.list_counts.capacity() * size_of::() + + self.list_id_bytes_lens.capacity() * size_of::() + + self.list_sqs.capacity() * size_of::() + + std::iter::once(&self.sq) + .chain(&self.list_sqs) + .map(|sq| (sq.mins.capacity() + sq.maxs.capacity()) * size_of::()) + .sum::(); + self.list_cache = + SqListCache::new(self.nlist, memory_budget_bytes.saturating_sub(resident)); + } + pub fn open(mut reader: R) -> io::Result { let mut header = [0u8; IVF_SQ_HEADER_SIZE]; reader.pread(&mut [ReadRequest::new(0, &mut header)])?; @@ -352,6 +402,7 @@ impl IVFSQIndexReader { list_counts, list_id_bytes_lens, loaded: true, + list_cache: None, }) } @@ -429,6 +480,56 @@ impl IVFSQIndexReader { .collect() } + fn read_scan_lists(&mut self, list_ids: &[usize]) -> io::Result>> { + if self.list_cache.is_none() { + return self + .read_inverted_lists(list_ids) + .map(|lists| lists.into_iter().map(Arc::new).collect()); + } + let mut results = vec![None; list_ids.len()]; + let mut misses = Vec::new(); + for (position, &list_id) in list_ids.iter().enumerate() { + if list_id >= self.nlist { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "IVF-SQ list ID out of range", + )); + } + let cache = self.list_cache.as_mut().unwrap(); + if let Some(entry) = &cache.entries[list_id] { + if entry.offset == self.list_offsets[list_id] + && entry.count == self.list_counts[list_id] + && entry.id_bytes_len == self.list_id_bytes_lens[list_id] + && entry.d == self.d + { + results[position] = Some(Arc::clone(&entry.list)); + continue; + } + // Public reader metadata may have been edited by a low-level + // caller. Never serve a payload for a different range or shape. + cache.remove(list_id); + cache.order.retain(|&id| id != list_id); + } + misses.push((position, list_id)); + } + if !misses.is_empty() { + let missing_ids = misses.iter().map(|&(_, id)| id).collect::>(); + let loaded = self.read_inverted_lists(&missing_ids)?; + for ((position, list_id), list) in misses.into_iter().zip(loaded) { + let list = Arc::new(list); + self.list_cache.as_mut().unwrap().insert(CachedSqList { + offset: self.list_offsets[list_id], + count: self.list_counts[list_id], + id_bytes_len: self.list_id_bytes_lens[list_id], + d: self.d, + list: Arc::clone(&list), + }); + results[position] = Some(list); + } + } + Ok(results.into_iter().map(Option::unwrap).collect()) + } + fn batch_read_end(&self, list_ids: &[usize]) -> io::Result { let payload_lengths = list_ids .iter() @@ -563,16 +664,28 @@ impl IVFSQIndexReader { } let count = self.batch_read_end(&probe_indices[batch_start..])?.max(1); let batch_end = (batch_start + count).min(probe_indices.len()); - let lists = self.read_inverted_lists(&probe_indices[batch_start..batch_end])?; + let lists = self.read_scan_lists(&probe_indices[batch_start..batch_end])?; let centroids = &self.quantizer_centroids; let list_sqs = &self.list_sqs; let global_sq = &self.sq; let candidate_count = lists.iter().map(|list| list.ids.len()).sum::(); if candidate_count >= PARALLEL_SQ_SCAN_MIN_CANDIDATES { - let per_list_results = lists + let first = &lists[0]; + scan_sq_list( + &query, + first, + ¢roids[first.list_id * d..(first.list_id + 1) * d], + list_sqs.get(first.list_id).unwrap_or(global_sq), + metric, + filter, + &mut SqScanScratch::default(), + &mut heap, + ); + let cutoff = heap.distance_limit(); + let per_list_results = lists[1..] .par_iter() .map_init(SqScanScratch::default, |scratch, list| { - let mut local_heap = TopKHeap::new(k); + let mut local_heap = TopKHeap::with_max_distance(k, cutoff); let list_id = list.list_id; scan_sq_list( &query, @@ -666,6 +779,9 @@ pub(crate) fn search_batch_ivfsq_reader_filter_range( )); } validate_batch_seed(seed_ids, seed_distances, nq, k)?; + if nq == 1 && probe_start == 0 && seed_ids.is_empty() { + return reader.search_with_filter(queries, k, probe_end, filter); + } let processed = preprocess_vectors(queries, nq, reader.d, reader.metric); let (all_probe_indices, _) = kmeans::find_topk_batch( &processed, @@ -731,41 +847,36 @@ pub(crate) fn search_batch_ivfsq_reader_filter_range( } let count = reader.batch_read_end(&unique_lists[batch_start..])?.max(1); let batch_end = (batch_start + count).min(unique_lists.len()); - let loaded_lists = reader.read_inverted_lists(&unique_lists[batch_start..batch_end])?; + let loaded_lists = reader.read_scan_lists(&unique_lists[batch_start..batch_end])?; let centroids = &reader.quantizer_centroids; let list_sqs = &reader.list_sqs; let global_sq = &reader.sq; - let per_list_results = loaded_lists - .par_iter() - .map_init(SqScanScratch::default, |scratch, list| { - let list_id = list.list_id; - list_to_queries[list_id] - .iter() - .map(|&query_index| { - let query = &processed[query_index * d..(query_index + 1) * d]; - let mut heap = TopKHeap::new(k); + let mut list_positions = vec![None; reader.nlist]; + for (position, list) in loaded_lists.iter().enumerate() { + list_positions[list.list_id] = Some(position); + } + // Keep a query's heap across partitions. Besides avoiding nprobe + // allocations and merges, this carries the current cutoff into later scans. + heaps.par_iter_mut().enumerate().for_each_init( + SqScanScratch::default, + |scratch, (query_index, heap)| { + let query = &processed[query_index * d..(query_index + 1) * d]; + for &list_id in all_probe_indices[query_index].iter().skip(probe_start) { + if let Some(position) = list_positions[list_id] { scan_sq_list( query, - list, + &loaded_lists[position], ¢roids[list_id * d..(list_id + 1) * d], list_sqs.get(list_id).unwrap_or(global_sq), metric, filter, scratch, - &mut heap, + heap, ); - (query_index, heap.into_sorted()) - }) - .collect::>() - }) - .collect::>(); - for list_results in per_list_results { - for (query_index, results) in list_results { - for (distance, row_id) in results { - heaps[query_index].push(distance, row_id); + } } - } - } + }, + ); batch_start = batch_end; } @@ -868,6 +979,75 @@ pub struct SqListData { pub codes: Vec, } +struct CachedSqList { + offset: i64, + count: i32, + id_bytes_len: i32, + d: usize, + list: Arc, +} + +impl CachedSqList { + fn retained_bytes(&self) -> usize { + size_of::() + + 2 * size_of::() + + self.list.ids.capacity() * size_of::() + + self.list.codes.capacity() + } +} + +/// FIFO eviction keeps cache bookkeeping O(1) without per-hit allocations. +/// Both the slot table and queue are reserved and charged to the budget up front. +struct SqListCache { + entries: Vec>, + order: VecDeque, + capacity_bytes: usize, + retained_bytes: usize, +} + +impl SqListCache { + fn new(nlist: usize, budget: usize) -> Option { + let fixed = nlist.checked_mul(size_of::>() + size_of::())?; + if budget <= fixed { + return None; + } + let entries = (0..nlist).map(|_| None).collect::>(); + let order = VecDeque::::with_capacity(nlist); + let fixed = entries.capacity() * size_of::>() + + order.capacity() * size_of::(); + if budget <= fixed { + return None; + } + Some(Self { + entries, + order, + capacity_bytes: budget.saturating_sub(fixed), + retained_bytes: 0, + }) + } + + fn remove(&mut self, list_id: usize) { + if let Some(entry) = self.entries[list_id].take() { + self.retained_bytes -= entry.retained_bytes(); + } + } + + fn insert(&mut self, entry: CachedSqList) { + let bytes = entry.retained_bytes(); + let list_id = entry.list.list_id; + if bytes > self.capacity_bytes || self.entries[list_id].is_some() { + return; + } + while bytes > self.capacity_bytes - self.retained_bytes { + let oldest = self.order.pop_front().expect("nonempty cache over budget"); + self.remove(oldest); + } + self.retained_bytes += bytes; + self.order.push_back(list_id); + self.entries[list_id] = Some(entry); + } +} + #[derive(Clone, Copy)] struct BatchedListRead { input_index: usize, @@ -929,6 +1109,7 @@ fn scan_sq_rows( centroid, metric, IVF_SQ_SCAN_BLOCK_SIZE, + heap.distance_limit(), &mut scratch.parameters, &mut scratch.distances, ); @@ -1083,13 +1264,16 @@ fn block_sorted_sq_codes( block_size: usize, ) -> Vec { debug_assert_eq!(row_major.len(), order.len() * d); - let mut blocked = Vec::with_capacity(row_major.len()); + let mut blocked = vec![0; row_major.len()]; for block_start in (0..order.len()).step_by(block_size) { let block_len = (order.len() - block_start).min(block_size); - for dimension in 0..d { - for lane in 0..block_len { - let source_row = order[block_start + lane]; - blocked.push(row_major[source_row * d + dimension]); + let block = &mut blocked[block_start * d..(block_start + block_len) * d]; + for (dimension, column) in block.chunks_exact_mut(block_len).enumerate() { + for (dst, &source_row) in column + .iter_mut() + .zip(&order[block_start..block_start + block_len]) + { + *dst = row_major[source_row * d + dimension]; } } } @@ -1123,6 +1307,121 @@ mod tests { use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::Arc; + #[test] + fn ivfsq_partition_cache_reuses_payloads_and_keeps_filters_query_local() { + let (index, data, _) = build_index(37, 8, 4_097); + let bytes = serialized_index(&index); + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(bytes.clone()), + calls: Arc::clone(&calls), + }; + let mut cached = IVFSQIndexReader::open_with_options( + source, + VectorIndexReaderOptions::new(4 * 1024 * 1024), + ) + .unwrap(); + let mut uncached = IVFSQIndexReader::open(Cursor::new(bytes)).unwrap(); + let query = &data[127 * 37..128 * 37]; + let expected = uncached.search(query, 10, 8).unwrap(); + assert_eq!(cached.search(query, 10, 8).unwrap(), expected); + calls.store(0, Ordering::Relaxed); + let first = cached.read_scan_lists(&[0, 1]).unwrap(); + let again = cached.read_scan_lists(&[1, 0]).unwrap(); + assert!(Arc::ptr_eq(&first[0], &again[1])); + assert!(Arc::ptr_eq(&first[1], &again[0])); + let filter = std::collections::HashSet::from([expected.0[3]]); + assert_eq!( + cached + .search_with_filter(query, 10, 8, Some(&filter)) + .unwrap(), + uncached + .search_with_filter(query, 10, 8, Some(&filter)) + .unwrap() + ); + assert_eq!(cached.search(query, 10, 8).unwrap(), expected); + assert_eq!( + calls.load(Ordering::Relaxed), + 0, + "warm scans must not reenter positional I/O" + ); + } + + #[test] + fn ivfsq_partition_cache_is_bounded_and_evicts_without_retaining_duplicate_keys() { + fn entry(list_id: usize, bytes: usize) -> CachedSqList { + CachedSqList { + offset: list_id as i64, + count: 1, + id_bytes_len: 1, + d: bytes, + list: Arc::new(SqListData { + list_id, + ids: vec![list_id as i64], + codes: vec![0; bytes], + }), + } + } + let fixed = 4 * (size_of::>() + size_of::()); + let size = entry(0, 64).retained_bytes(); + let mut cache = SqListCache::new(4, fixed + size * 2).unwrap(); + cache.insert(entry(0, 64)); + cache.insert(entry(1, 64)); + cache.insert(entry(1, 64)); + assert_eq!(cache.order.iter().copied().collect::>(), [0, 1]); + cache.insert(entry(2, 64)); + assert!(cache.entries[0].is_none()); + assert!(cache.entries[1].is_some()); + assert!(cache.entries[2].is_some()); + assert_eq!(cache.retained_bytes, size * 2); + cache.insert(entry(3, size * 3)); + assert!( + cache.entries[3].is_none(), + "an oversized partition must bypass the cache" + ); + assert_eq!(cache.retained_bytes, size * 2); + assert!(SqListCache::new(4, 0).is_none()); + assert!(SqListCache::new(4, fixed).is_none()); + } + + #[test] + fn ivfsq_cache_misses_retry_after_io_failure_and_zero_budget_stays_uncached() { + let (index, _, _) = build_index(5, 2, 65); + let bytes = serialized_index(&index); + let mut cached = IVFSQIndexReader::open_with_options( + Cursor::new(bytes.clone()), + VectorIndexReaderOptions::new(1024 * 1024), + ) + .unwrap(); + cached.read_scan_lists(&[0]).unwrap(); + let offset = cached.list_offsets[1]; + cached.list_offsets[1] = bytes.len() as i64 + 1; + assert!(cached.read_scan_lists(&[1]).is_err()); + assert!(cached.list_cache.as_ref().unwrap().entries[1].is_none()); + cached.list_offsets[1] = offset; + let expected = cached.read_scan_lists(&[1]).unwrap(); + // Even a warmed entry must not hide an edited offset. + cached.list_offsets[1] = bytes.len() as i64 + 1; + assert!(cached.read_scan_lists(&[1]).is_err()); + cached.list_offsets[1] = offset; + assert_eq!( + cached.read_scan_lists(&[1]).unwrap()[0].ids, + expected[0].ids + ); + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(bytes), + calls: Arc::clone(&calls), + }; + let mut uncached = + IVFSQIndexReader::open_with_options(source, VectorIndexReaderOptions::new(0)).unwrap(); + assert!(uncached.list_cache.is_none()); + calls.store(0, Ordering::Relaxed); + uncached.read_scan_lists(&[0, 1]).unwrap(); + uncached.read_scan_lists(&[0, 1]).unwrap(); + assert_eq!(calls.load(Ordering::Relaxed), 2); + } + fn build_index(d: usize, nlist: usize, n: usize) -> (IVFSQIndex, Vec, Vec) { let data = (0..n) .flat_map(|i| { @@ -1189,6 +1488,48 @@ mod tests { assert_eq!(batch.1, [first.1, second.1].concat()); } + #[test] + fn ivfsq_single_query_batch_and_seeded_probe_ranges_match_full_search() { + let d = 37; + let nlist = 8; + let (index, data, _) = build_index(d, nlist, 8_193); + let bytes = serialized_index(&index); + let mut reader = IVFSQIndexReader::open(Cursor::new(bytes)).unwrap(); + let query = &data[127 * d..128 * d]; + let expected = reader.search(query, 10, nlist).unwrap(); + let one = search_batch_ivfsq_reader(&mut reader, query, 1, 10, nlist).unwrap(); + assert_eq!(one, expected); + let first = + search_batch_ivfsq_reader_filter_range(&mut reader, query, 1, 10, 0, 3, &[], &[], None) + .unwrap(); + let refined = search_batch_ivfsq_reader_filter_range( + &mut reader, + query, + 1, + 10, + 3, + nlist, + &first.0, + &first.1, + None, + ) + .unwrap(); + // Repeated vectors can tie; verify distances and the returned IDs' scores. + assert_eq!(refined.1, expected.1); + for (&id, &distance) in refined.0.iter().zip(&refined.1) { + let (ids, distances) = reader + .search_with_filter( + query, + 1, + nlist, + Some(&std::collections::HashSet::from([id])), + ) + .unwrap(); + assert_eq!(ids, [id]); + assert_eq!(distances, [distance]); + } + } + #[test] fn ivfsq_large_batch_scans_queries_in_parallel_without_duplicate_reads() { let d = 16; diff --git a/core/src/sq.rs b/core/src/sq.rs index 0785055..ffe58be 100644 --- a/core/src/sq.rs +++ b/core/src/sq.rs @@ -101,6 +101,54 @@ impl ScalarQuantizer { self.encode_batch(vector, 1, code); } + /// Gather and quantize a partition without materializing its residual matrix. + pub(crate) fn encode_residual_rows( + &self, + data: &[f32], + rows: &[usize], + offset: &[f32], + codes: &mut [u8], + ) { + assert_eq!(codes.len(), rows.len() * self.d); + assert_eq!(offset.len(), self.d); + assert_eq!(self.mins.len(), self.d); + assert_eq!(self.maxs.len(), self.d); + let scales = self + .mins + .iter() + .zip(&self.maxs) + .map( + |(&min, &max)| { + if min < max { + 255.0 / (max - min) + } else { + 0.0 + } + }, + ) + .collect::>(); + for (&row, code) in rows.iter().zip(codes.chunks_exact_mut(self.d)) { + let vector = &data[row * self.d..(row + 1) * self.d]; + encode_residual(vector, offset, &self.mins, &self.maxs, &scales, code); + } + } + + pub(crate) fn train_residual_rows(data: &[f32], rows: &[usize], offset: &[f32]) -> Self { + let d = offset.len(); + let mut mins = vec![f32::INFINITY; d]; + let mut maxs = vec![f32::NEG_INFINITY; d]; + // Subtraction is monotone: subtracting the centroid from the extrema + // gives exactly the extrema of the rounded residuals, with O(d) scratch. + for &row in rows { + update_bounds_batch(&data[row * d..(row + 1) * d], 1, d, &mut mins, &mut maxs); + } + for dim in 0..d { + mins[dim] -= offset[dim]; + maxs[dim] -= offset[dim]; + } + Self::with_dimension_bounds(d, mins, maxs) + } + pub fn decode_batch(&self, codes: &[u8], n: usize, vectors: &mut [f32]) { let len = n * self.d; assert!(codes.len() >= len); @@ -209,6 +257,7 @@ impl ScalarQuantizer { offset: &[f32], metric: MetricType, block_size: usize, + cutoff: f32, parameters: &mut Vec, distances: &mut Vec, ) { @@ -226,7 +275,9 @@ impl ScalarQuantizer { primary[dimension] = query[dimension] - offset[dimension] - self.mins[dimension]; scales[dimension] = (self.maxs[dimension] - self.mins[dimension]) * (1.0 / 255.0); } - blocked_sq_l2(primary, scales, codes, count, self.d, block_size, distances); + blocked_sq_l2( + primary, scales, codes, count, self.d, block_size, cutoff, distances, + ); return; } @@ -426,21 +477,24 @@ fn blocked_sq_l2( count: usize, d: usize, block_size: usize, + cutoff: f32, distances: &mut [f32], ) { #[cfg(target_arch = "x86_64")] if block_size == 32 && is_x86_feature_detected!("avx2") { unsafe { - return blocked_sq_l2_avx2(biases, scales, codes, count, d, distances); + return blocked_sq_l2_avx2(biases, scales, codes, count, d, cutoff, distances); } } #[cfg(target_arch = "aarch64")] if block_size == 32 { unsafe { - return blocked_sq_l2_neon(biases, scales, codes, count, d, distances); + return blocked_sq_l2_neon(biases, scales, codes, count, d, cutoff, distances); } } - blocked_sq_l2_scalar(biases, scales, codes, count, d, block_size, distances); + blocked_sq_l2_scalar( + biases, scales, codes, count, d, block_size, cutoff, distances, + ); } fn blocked_sq_l2_scalar( @@ -450,6 +504,7 @@ fn blocked_sq_l2_scalar( count: usize, d: usize, block_size: usize, + cutoff: f32, distances: &mut [f32], ) { let mut code_offset = 0usize; @@ -457,14 +512,20 @@ fn blocked_sq_l2_scalar( let block_len = (count - block_start).min(block_size); let block_distances = &mut distances[block_start..block_start + block_len]; block_distances.fill(0.0); - for dimension in 0..d { - let column = &codes - [code_offset + dimension * block_len..code_offset + (dimension + 1) * block_len]; - let bias = biases[dimension]; - let scale = scales[dimension]; - for lane in 0..block_len { - let difference = bias - column[lane] as f32 * scale; - block_distances[lane] += difference * difference; + let checkpoint = (d / 2).max(32).min(d); + for (start, end) in [(0, checkpoint), (checkpoint, d)] { + if start > 0 && block_distances.iter().all(|&distance| distance >= cutoff) { + break; + } + for dimension in start..end { + let column = &codes[code_offset + dimension * block_len + ..code_offset + (dimension + 1) * block_len]; + let bias = biases[dimension]; + let scale = scales[dimension]; + for lane in 0..block_len { + let difference = bias - column[lane] as f32 * scale; + block_distances[lane] += difference * difference; + } } } code_offset += block_len * d; @@ -479,6 +540,7 @@ unsafe fn blocked_sq_l2_neon( codes: &[u8], count: usize, d: usize, + cutoff: f32, distances: &mut [f32], ) { use std::arch::aarch64::*; @@ -487,23 +549,29 @@ unsafe fn blocked_sq_l2_neon( for block_start in (0..full_count).step_by(32) { let code_base = block_start * d; let mut accumulators = [vdupq_n_f32(0.0); 8]; - for dimension in 0..d { - let column = codes.as_ptr().add(code_base + dimension * 32); - let bias = vdupq_n_f32(biases[dimension]); - let scale = vdupq_n_f32(scales[dimension]); - for chunk in 0..4 { - let code_u16 = vmovl_u8(vld1_u8(column.add(chunk * 8))); - let code_low = vcvtq_f32_u32(vmovl_u16(vget_low_u16(code_u16))); - let code_high = vcvtq_f32_u32(vmovl_u16(vget_high_u16(code_u16))); - let difference_low = vfmsq_f32(bias, code_low, scale); - let difference_high = vfmsq_f32(bias, code_high, scale); - accumulators[chunk * 2] = - vfmaq_f32(accumulators[chunk * 2], difference_low, difference_low); - accumulators[chunk * 2 + 1] = vfmaq_f32( - accumulators[chunk * 2 + 1], - difference_high, - difference_high, - ); + let checkpoint = (d / 2).max(32).min(d); + for (start, end) in [(0, checkpoint), (checkpoint, d)] { + if start > 0 && accumulators.iter().all(|&acc| vminvq_f32(acc) >= cutoff) { + break; + } + for dimension in start..end { + let column = codes.as_ptr().add(code_base + dimension * 32); + let bias = vdupq_n_f32(biases[dimension]); + let scale = vdupq_n_f32(scales[dimension]); + for chunk in 0..4 { + let code_u16 = vmovl_u8(vld1_u8(column.add(chunk * 8))); + let code_low = vcvtq_f32_u32(vmovl_u16(vget_low_u16(code_u16))); + let code_high = vcvtq_f32_u32(vmovl_u16(vget_high_u16(code_u16))); + let difference_low = vfmsq_f32(bias, code_low, scale); + let difference_high = vfmsq_f32(bias, code_high, scale); + accumulators[chunk * 2] = + vfmaq_f32(accumulators[chunk * 2], difference_low, difference_low); + accumulators[chunk * 2 + 1] = vfmaq_f32( + accumulators[chunk * 2 + 1], + difference_high, + difference_high, + ); + } } } for (chunk, accumulator) in accumulators.into_iter().enumerate() { @@ -521,6 +589,7 @@ unsafe fn blocked_sq_l2_neon( count - full_count, d, 32, + cutoff, &mut distances[full_count..], ); } @@ -534,6 +603,7 @@ unsafe fn blocked_sq_l2_avx2( codes: &[u8], count: usize, d: usize, + cutoff: f32, distances: &mut [f32], ) { use std::arch::x86_64::*; @@ -542,15 +612,27 @@ unsafe fn blocked_sq_l2_avx2( for block_start in (0..full_count).step_by(32) { let code_base = block_start * d; let mut accumulators = [_mm256_setzero_ps(); 4]; - for dimension in 0..d { - let column = codes.as_ptr().add(code_base + dimension * 32); - let bias = _mm256_set1_ps(biases[dimension]); - let scale = _mm256_set1_ps(scales[dimension]); - for (chunk, accumulator) in accumulators.iter_mut().enumerate() { - let bytes = _mm_loadl_epi64(column.add(chunk * 8).cast()); - let code = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(bytes)); - let difference = _mm256_sub_ps(bias, _mm256_mul_ps(code, scale)); - *accumulator = _mm256_add_ps(*accumulator, _mm256_mul_ps(difference, difference)); + let checkpoint = (d / 2).max(32).min(d); + for (start, end) in [(0, checkpoint), (checkpoint, d)] { + if start > 0 + && accumulators.iter().all(|&acc| { + _mm256_movemask_ps(_mm256_cmp_ps::<_CMP_GE_OQ>(acc, _mm256_set1_ps(cutoff))) + == 255 + }) + { + break; + } + for dimension in start..end { + let column = codes.as_ptr().add(code_base + dimension * 32); + let bias = _mm256_set1_ps(biases[dimension]); + let scale = _mm256_set1_ps(scales[dimension]); + for (chunk, accumulator) in accumulators.iter_mut().enumerate() { + let bytes = _mm_loadl_epi64(column.add(chunk * 8).cast()); + let code = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(bytes)); + let difference = _mm256_sub_ps(bias, _mm256_mul_ps(code, scale)); + *accumulator = + _mm256_add_ps(*accumulator, _mm256_mul_ps(difference, difference)); + } } } for (chunk, accumulator) in accumulators.into_iter().enumerate() { @@ -568,6 +650,7 @@ unsafe fn blocked_sq_l2_avx2( count - full_count, d, 32, + cutoff, &mut distances[full_count..], ); } @@ -870,6 +953,131 @@ unsafe fn update_bounds_batch_neon( } } +fn encode_residual( + vector: &[f32], + offset: &[f32], + mins: &[f32], + maxs: &[f32], + scales: &[f32], + codes: &mut [u8], +) { + #[cfg(target_arch = "aarch64")] + let start = unsafe { encode_residual_neon(vector, offset, mins, scales, codes) }; + #[cfg(target_arch = "x86_64")] + let start = if is_x86_feature_detected!("avx2") { + unsafe { encode_residual_avx2(vector, offset, mins, scales, codes) } + } else { + 0 + }; + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + let start = { + let _ = scales; + 0 + }; + for dim in start..codes.len() { + codes[dim] = encode_value(vector[dim] - offset[dim], mins[dim], maxs[dim]); + } +} + +#[cfg(target_arch = "aarch64")] +#[target_feature(enable = "neon")] +unsafe fn encode_residual_neon( + vector: &[f32], + offset: &[f32], + mins: &[f32], + scales: &[f32], + codes: &mut [u8], +) -> usize { + use std::arch::aarch64::*; + let mut dim = 0; + while dim + 16 <= codes.len() { + let mut packed = [vdupq_n_u32(0); 4]; + for (chunk, out) in packed.iter_mut().enumerate() { + let i = dim + chunk * 4; + let residual = vsubq_f32( + vld1q_f32(vector.as_ptr().add(i)), + vld1q_f32(offset.as_ptr().add(i)), + ); + let value = vmulq_f32( + vsubq_f32(residual, vld1q_f32(mins.as_ptr().add(i))), + vld1q_f32(scales.as_ptr().add(i)), + ); + *out = vcvtaq_u32_f32(vminq_f32( + vdupq_n_f32(255.0), + vmaxq_f32(vdupq_n_f32(0.0), value), + )); + } + let low = vcombine_u16(vmovn_u32(packed[0]), vmovn_u32(packed[1])); + let high = vcombine_u16(vmovn_u32(packed[2]), vmovn_u32(packed[3])); + vst1q_u8( + codes.as_mut_ptr().add(dim), + vcombine_u8(vmovn_u16(low), vmovn_u16(high)), + ); + dim += 16; + } + // Keep the same four-dimension SIMD boundary as encode_batch. + while dim + 4 <= codes.len() { + let residual = vsubq_f32( + vld1q_f32(vector.as_ptr().add(dim)), + vld1q_f32(offset.as_ptr().add(dim)), + ); + let value = vmulq_f32( + vsubq_f32(residual, vld1q_f32(mins.as_ptr().add(dim))), + vld1q_f32(scales.as_ptr().add(dim)), + ); + let rounded = vcvtaq_u32_f32(vminq_f32( + vdupq_n_f32(255.0), + vmaxq_f32(vdupq_n_f32(0.0), value), + )); + let bytes = vmovn_u16(vcombine_u16(vmovn_u32(rounded), vdup_n_u16(0))); + codes[dim..dim + 4] + .copy_from_slice(&vget_lane_u32::<0>(vreinterpret_u32_u8(bytes)).to_le_bytes()); + dim += 4; + } + dim +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn encode_residual_avx2( + vector: &[f32], + offset: &[f32], + mins: &[f32], + scales: &[f32], + codes: &mut [u8], +) -> usize { + use std::arch::x86_64::*; + let mut dim = 0; + while dim + 8 <= codes.len() { + let residual = _mm256_sub_ps( + _mm256_loadu_ps(vector.as_ptr().add(dim)), + _mm256_loadu_ps(offset.as_ptr().add(dim)), + ); + let value = _mm256_mul_ps( + _mm256_sub_ps(residual, _mm256_loadu_ps(mins.as_ptr().add(dim))), + _mm256_loadu_ps(scales.as_ptr().add(dim)), + ); + let value = _mm256_min_ps( + _mm256_set1_ps(255.0), + _mm256_max_ps(_mm256_setzero_ps(), value), + ); + let floor = _mm256_floor_ps(value); + let up = _mm256_cmp_ps::<_CMP_GE_OQ>(_mm256_sub_ps(value, floor), _mm256_set1_ps(0.5)); + let rounded = + _mm256_cvttps_epi32(_mm256_add_ps(floor, _mm256_and_ps(up, _mm256_set1_ps(1.0)))); + let words = _mm_packus_epi32( + _mm256_castsi256_si128(rounded), + _mm256_extracti128_si256::<1>(rounded), + ); + _mm_storel_epi64( + codes.as_mut_ptr().add(dim).cast(), + _mm_packus_epi16(words, words), + ); + dim += 8; + } + dim +} + fn encode_batch_simd( data: &[f32], n: usize, @@ -1286,6 +1494,129 @@ impl DistanceContext { mod tests { use super::*; + #[test] + fn residual_encoder_matches_separate_subtraction_and_encoding() { + for d in [1, 3, 4, 7, 8, 15, 16, 17, 31, 32, 37, 128] { + let mins = (0..d) + .map(|j| if j % 5 == 0 { 7.0 } else { -2.0 }) + .collect::>(); + let maxs = (0..d) + .map(|j| if j % 5 == 0 { 7.0 } else { 253.0 }) + .collect::>(); + let sq = ScalarQuantizer::with_dimension_bounds(d, mins, maxs); + let offset = (0..d).map(|j| j as f32 * 0.25 - 4.0).collect::>(); + // Includes clipping, exact half-way rounding, and constant dimensions. + let levels = [-10.0, -2.0, -1.5, 0.5, 127.5, 252.5, 253.0, 260.0]; + let data = (0..levels.len() * d) + .map(|i| levels[(i / d + i % d) % levels.len()] + offset[i % d]) + .collect::>(); + let rows = [7, 0, 3, 2, 3, 6, 1, 5, 4]; + let mut actual = vec![99; rows.len() * d]; + sq.encode_residual_rows(&data, &rows, &offset, &mut actual); + let mut expected = vec![0; actual.len()]; + for (&row, code) in rows.iter().zip(expected.chunks_exact_mut(d)) { + let residual = (0..d) + .map(|j| data[row * d + j] - offset[j]) + .collect::>(); + sq.encode(&residual, code); + } + assert_eq!(actual, expected, "dimension {d}"); + } + } + + #[test] + fn residual_extrema_match_materialized_residuals() { + let d = 37; + let data = (0..19 * d) + .map(|i| ((i * 17 % 131) as f32 - 65.0) * 0.031) + .collect::>(); + let offset = (0..d).map(|j| j as f32 * 0.17 - 1.0).collect::>(); + let rows = [18, 1, 3, 7, 0, 1]; + let actual = ScalarQuantizer::train_residual_rows(&data, &rows, &offset); + let residuals = rows + .iter() + .flat_map(|&row| { + (0..d) + .map(|j| data[row * d + j] - offset[j]) + .collect::>() + }) + .collect::>(); + let mut expected = ScalarQuantizer::new(d); + expected.train(&residuals, rows.len()); + assert_eq!(actual.mins, expected.mins); + assert_eq!(actual.maxs, expected.maxs); + } + + #[test] + fn blocked_l2_cutoff_preserves_every_competitive_distance() { + for d in [1, 31, 32, 33, 64, 65, 128] { + for count in [1, 31, 32, 33, 63, 64, 65, 97] { + for block_size in [7, 32] { + let sq = ScalarQuantizer::with_bounds(d, 0.0, 255.0); + let query = vec![0.0; d]; + let mut codes = Vec::new(); + for start in (0..count).step_by(block_size) { + let len = (count - start).min(block_size); + for dim in 0..d { + for lane in 0..len { + // One entire far block, and a lane whose distance + // grows only in the last dimension, guard both sides. + codes.push(if start < 32 { + 20 + } else if lane == 0 { + if dim + 1 == d { + 10 + } else { + 0 + } + } else { + 1 + }); + } + } + } + let mut parameters = Vec::new(); + let mut exact = Vec::new(); + sq.distances_to_blocked_codes_with_offset( + &query, + &codes, + count, + &query, + MetricType::L2, + block_size, + f32::INFINITY, + &mut parameters, + &mut exact, + ); + for cutoff in [0.0, 50.0, 100.0, 1000.0, f32::INFINITY] { + let mut actual = Vec::new(); + sq.distances_to_blocked_codes_with_offset( + &query, + &codes, + count, + &query, + MetricType::L2, + block_size, + cutoff, + &mut parameters, + &mut actual, + ); + for (&full, &pruned) in exact.iter().zip(&actual) { + if full < cutoff { + assert_eq!(pruned, full); + } else { + assert!( + pruned >= cutoff, + "unsafe cutoff d={d}, count={count}, block={block_size}" + ); + } + } + } + } + } + } + } + #[test] fn test_scalar_quantizer_round_trips_bounds() { let data = vec![-1.0, 0.0, 1.0, 3.0]; diff --git a/core/src/topk.rs b/core/src/topk.rs index ae7bfc6..c600af1 100644 --- a/core/src/topk.rs +++ b/core/src/topk.rs @@ -86,6 +86,10 @@ impl TopKHeap { self.is_full().then(|| self.data[0].0) } + pub(crate) fn distance_limit(&self) -> f32 { + self.worst_distance().unwrap_or(self.max_distance) + } + pub(crate) fn into_sorted(mut self) -> Vec<(f32, i64)> { self.data.sort_by(|a, b| a.0.total_cmp(&b.0)); self.data diff --git a/docs/api.html b/docs/api.html index 0be8fbb..b1e004d 100644 --- a/docs/api.html +++ b/docs/api.html @@ -68,27 +68,28 @@

Shared search parameters

-

Reader options for DiskANN

-

Reader options are accepted by every binding and affect DiskANN only. Other index families continue to use their existing positional-I/O behavior.

+

Reader options for DiskANN and IVF-SQ

+

Reader options are accepted by every binding. The memory budget controls DiskANN resident state and caches, and IVF-SQ's decoded-partition cache. The latency hint and read-plan inspection described below are DiskANN-specific.

- +
ConceptRust / PythonC / C++ / JavaDefault
Random-read latency hintInput capability estimated_random_read_latency_nanosC/C++ field / Java estimatedRandomReadLatencyNanos()0: reuse the mandatory header read's elapsed time; positive values bypass measurement
Total Reader memorymemory_budget_bytesmemory_budget_bytes / constructor argument4 GiB; automatically partitioned among required resident state, a profile-sized adjacency prefix, a bounded cold-adjacency LRU, and a bounded raw-vector LRU
Total Reader memorymemory_budget_bytesmemory_budget_bytes / constructor argument4 GiB; DiskANN divides it among resident state and adjacency/raw-vector caches. IVF-SQ reserves resident metadata first and uses the remainder for a bounded decoded-partition FIFO cache; zero disables that cache.
+

For IVF-SQ, the unified Rust reader's open and binding constructors use the default budget; open_with_options allows an override. Direct IVFSQIndexReader::open is uncached, while its open_with_options enables caching. A zero or insufficient budget disables retention, but required metadata still loads. The limit accounts for resident metadata and retained cache allocations, not transient query buffers or storage-adapter allocations. See IVF-SQ cache behavior.

The Reader has no public storage-tier switch. DiskANN selects an internal tier once during open from the latency hint or mandatory header-read timing; its latency, window, and beam policy then remain stable. read_plan / readPlan exposes that policy together with the current effective preload and shared-cache capacities. Those capacities are zero before resident initialization and may shrink when lazy row-ID lookup state consumes the same total budget. Cache partitioning remains an internal decision. The input callback receives all positional ranges for one round together and should execute them concurrently.

A storage adapter may additionally advertise preferred_window_bytes and a maximum range count named max_ranges_per_pread in Rust or max_ranges_per_read in C/C++/Python. Zero means unspecified. DiskANN rounds the requested window to complete 4 KiB logical pages, bounds it to 1 MiB, limits each pread batch to the advertised range count, and keeps physical alignment concerns inside the storage adapter. Build-time deployment-profile remains separate because it can select the persisted compact or interleaved layout. See DiskANN reader tuning.

Search warm-up

-

After opening a Reader and before repeated searches, initialize resident state and, for DiskANN, optionally replay a small representative query set. Warm-up builds process-local caches without changing the file or search results.

-
+

After opening a Reader and before repeated searches, initialize resident state and optionally replay a small representative query set. DiskANN exposes a query warm-up method; IVF-SQ populates its partition cache through ordinary search or batch-search calls. Warm-up builds process-local caches without changing the file or search results.

+
LanguageResident initializationRepresentative-query warm-upDiskANN width calibration
LanguageResident initializationDiskANN query warm-upDiskANN width calibration
Rustoptimize_for_searchwarmup_queriescalibrate_search_width
Cpaimon_vindex_reader_optimize_for_searchpaimon_vindex_reader_warmup_queriespaimon_vindex_reader_calibrate_search_width
C++optimize_for_searchwarmup_queriescalibrate_search_width
JavaoptimizeForSearchwarmupQueriescalibrateSearchWidth
Pythonoptimize_for_searchwarmup_queriescalibrate_search_width
-

optimize_for_search is optional for correctness: the first search performs the same initialization lazily. It builds IVF-PQ residual-L2 tables or DiskANN resident PQ/row-ID state and its automatically budgeted hot adjacency prefix. DiskANN warmup_queries then executes top-1 graph traversal and persisted-vector rerank for each supplied query, priming immutable adjacency and raw-vector LRUs. Other index families treat representative warm-up as resident initialization only.

+

optimize_for_search is optional for correctness: the first search performs the same initialization lazily. It builds IVF-PQ residual-L2 tables or DiskANN resident PQ/row-ID state and its automatically budgeted hot adjacency prefix. DiskANN warmup_queries then executes top-1 graph traversal and persisted-vector rerank for each supplied query, priming immutable adjacency and raw-vector LRUs. Other index families treat this method as resident initialization only. In particular, IVF-SQ metadata is already loaded at open; neither initialization nor warmup_queries loads its partition payloads. Replay queries at the intended nprobe through the search API to warm those payloads.

diff --git a/docs/benchmarks/ivfsq-lance-20260906.json b/docs/benchmarks/ivfsq-lance-20260906.json new file mode 100644 index 0000000..145e336 --- /dev/null +++ b/docs/benchmarks/ivfsq-lance-20260906.json @@ -0,0 +1,1372 @@ +{ + "baseline_commit": "8dcabf208c99707eab3938af0bcc40530fdb4cad", + "date": "2026-09-06", + "parameters": { + "nlist": 1024, + "nprobe": 64, + "k": 10, + "nq": 1000, + "train_n": 65536, + "threads": 8, + "repetitions": 3, + "lance_version": "11.0.0", + "paimon_memory_budget_bytes": 4294967296, + "lance_index_cache_bytes": 1073741824, + "lance_training": "native defaults; sample_rate=64", + "lance_cache_parameter": "index_cache_size_bytes" + }, + "paimon_rust": [ + { + "dataset": "sift", + "implementation": "baseline", + "repeat": 1, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 942, + "train_ms": 266, + "add_ms": 549, + "write_ms": 125, + "peak_rss_bytes": 830668800, + "file_bytes": 131359859, + "recall_at_10": 0.8626, + "first_query_us": 903, + "p50_query_us": 709, + "p95_query_us": 845, + "sequential_qps": 1389.47, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 8790731156, + "batch_ms": 158, + "batch_qps": 6294.27, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "baseline", + "repeat": 2, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 934, + "train_ms": 256, + "add_ms": 554, + "write_ms": 122, + "peak_rss_bytes": 834748416, + "file_bytes": 131359859, + "recall_at_10": 0.8626, + "first_query_us": 889, + "p50_query_us": 699, + "p95_query_us": 822, + "sequential_qps": 1418.89, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 8790731156, + "batch_ms": 155, + "batch_qps": 6416.66, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "baseline", + "repeat": 3, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 928, + "train_ms": 258, + "add_ms": 544, + "write_ms": 125, + "peak_rss_bytes": 836927488, + "file_bytes": 131359859, + "recall_at_10": 0.8626, + "first_query_us": 861, + "p50_query_us": 695, + "p95_query_us": 840, + "sequential_qps": 1412.83, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 8790731156, + "batch_ms": 154, + "batch_qps": 6459.88, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "optimized_uncached", + "repeat": 1, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 850, + "train_ms": 251, + "add_ms": 526, + "write_ms": 72, + "peak_rss_bytes": 817119232, + "file_bytes": 131359859, + "recall_at_10": 0.9811, + "first_query_us": 956, + "p50_query_us": 647, + "p95_query_us": 775, + "sequential_qps": 1529.28, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 8790731156, + "batch_ms": 108, + "batch_qps": 9175.67, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "optimized_uncached", + "repeat": 2, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 863, + "train_ms": 251, + "add_ms": 529, + "write_ms": 82, + "peak_rss_bytes": 823115776, + "file_bytes": 131359859, + "recall_at_10": 0.9811, + "first_query_us": 858, + "p50_query_us": 636, + "p95_query_us": 756, + "sequential_qps": 1557.09, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 8790731156, + "batch_ms": 111, + "batch_qps": 9001.14, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "optimized_uncached", + "repeat": 3, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 839, + "train_ms": 250, + "add_ms": 519, + "write_ms": 68, + "peak_rss_bytes": 815874048, + "file_bytes": 131359859, + "recall_at_10": 0.9811, + "first_query_us": 873, + "p50_query_us": 630, + "p95_query_us": 761, + "sequential_qps": 1560.51, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 8790731156, + "batch_ms": 104, + "batch_qps": 9527.79, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "optimized_cached", + "repeat": 1, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 886, + "train_ms": 253, + "add_ms": 538, + "write_ms": 93, + "peak_rss_bytes": 819658752, + "file_bytes": 131359859, + "recall_at_10": 0.9811, + "first_query_us": 895, + "p50_query_us": 193, + "p95_query_us": 300, + "sequential_qps": 4835.36, + "sequential_pread_rounds": 98, + "sequential_pread_bytes": 120400214, + "batch_ms": 110, + "batch_qps": 9046.32, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "optimized_cached", + "repeat": 2, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 879, + "train_ms": 251, + "add_ms": 540, + "write_ms": 87, + "peak_rss_bytes": 828325888, + "file_bytes": 131359859, + "recall_at_10": 0.9811, + "first_query_us": 903, + "p50_query_us": 194, + "p95_query_us": 298, + "sequential_qps": 4818.56, + "sequential_pread_rounds": 98, + "sequential_pread_bytes": 120400214, + "batch_ms": 111, + "batch_qps": 8986.74, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "sift", + "implementation": "optimized_cached", + "repeat": 3, + "n": 1000000, + "d": 128, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 902, + "train_ms": 252, + "add_ms": 521, + "write_ms": 106, + "peak_rss_bytes": 825360384, + "file_bytes": 131359859, + "recall_at_10": 0.9811, + "first_query_us": 936, + "p50_query_us": 204, + "p95_query_us": 275, + "sequential_qps": 4665.87, + "sequential_pread_rounds": 98, + "sequential_pread_bytes": 120400214, + "batch_ms": 122, + "batch_qps": 8153.26, + "batch_pread_rounds": 2, + "batch_pread_bytes": 129759867 + }, + { + "dataset": "glove", + "implementation": "baseline", + "repeat": 1, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 849, + "train_ms": 190, + "add_ms": 539, + "write_ms": 118, + "peak_rss_bytes": 756957184, + "file_bytes": 121822316, + "recall_at_10": 0.8036, + "first_query_us": 893, + "p50_query_us": 613, + "p95_query_us": 722, + "sequential_qps": 1602.82, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 7329440423, + "batch_ms": 141, + "batch_qps": 7042.59, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "baseline", + "repeat": 2, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 873, + "train_ms": 189, + "add_ms": 563, + "write_ms": 119, + "peak_rss_bytes": 763068416, + "file_bytes": 121822316, + "recall_at_10": 0.8036, + "first_query_us": 741, + "p50_query_us": 622, + "p95_query_us": 743, + "sequential_qps": 1569.14, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 7329440423, + "batch_ms": 143, + "batch_qps": 6988.29, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "baseline", + "repeat": 3, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 832, + "train_ms": 190, + "add_ms": 533, + "write_ms": 108, + "peak_rss_bytes": 762527744, + "file_bytes": 121822316, + "recall_at_10": 0.8036, + "first_query_us": 743, + "p50_query_us": 616, + "p95_query_us": 739, + "sequential_qps": 1580.78, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 7329440423, + "batch_ms": 144, + "batch_qps": 6939.38, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "optimized_uncached", + "repeat": 1, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 786, + "train_ms": 183, + "add_ms": 534, + "write_ms": 68, + "peak_rss_bytes": 773079040, + "file_bytes": 121822316, + "recall_at_10": 0.876, + "first_query_us": 755, + "p50_query_us": 564, + "p95_query_us": 681, + "sequential_qps": 1733.36, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 7329440423, + "batch_ms": 97, + "batch_qps": 10224.32, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "optimized_uncached", + "repeat": 2, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 781, + "train_ms": 183, + "add_ms": 526, + "write_ms": 69, + "peak_rss_bytes": 764608512, + "file_bytes": 121822316, + "recall_at_10": 0.876, + "first_query_us": 814, + "p50_query_us": 575, + "p95_query_us": 682, + "sequential_qps": 1702.6, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 7329440423, + "batch_ms": 98, + "batch_qps": 10102.09, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "optimized_uncached", + "repeat": 3, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 760, + "train_ms": 183, + "add_ms": 509, + "write_ms": 66, + "peak_rss_bytes": 771784704, + "file_bytes": 121822316, + "recall_at_10": 0.876, + "first_query_us": 688, + "p50_query_us": 571, + "p95_query_us": 673, + "sequential_qps": 1729.91, + "sequential_pread_rounds": 1000, + "sequential_pread_bytes": 7329440423, + "batch_ms": 93, + "batch_qps": 10697.93, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "optimized_cached", + "repeat": 1, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 803, + "train_ms": 187, + "add_ms": 526, + "write_ms": 90, + "peak_rss_bytes": 768311296, + "file_bytes": 121822316, + "recall_at_10": 0.876, + "first_query_us": 656, + "p50_query_us": 179, + "p95_query_us": 282, + "sequential_qps": 5208.74, + "sequential_pread_rounds": 112, + "sequential_pread_bytes": 113643808, + "batch_ms": 99, + "batch_qps": 10009.12, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "optimized_cached", + "repeat": 2, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 797, + "train_ms": 190, + "add_ms": 517, + "write_ms": 89, + "peak_rss_bytes": 772472832, + "file_bytes": 121822316, + "recall_at_10": 0.876, + "first_query_us": 665, + "p50_query_us": 179, + "p95_query_us": 282, + "sequential_qps": 5174.53, + "sequential_pread_rounds": 112, + "sequential_pread_bytes": 113643808, + "batch_ms": 97, + "batch_qps": 10296.44, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "glove", + "implementation": "optimized_cached", + "repeat": 3, + "n": 1183514, + "d": 100, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 793, + "train_ms": 185, + "add_ms": 516, + "write_ms": 91, + "peak_rss_bytes": 767737856, + "file_bytes": 121822316, + "recall_at_10": 0.876, + "first_query_us": 739, + "p50_query_us": 187, + "p95_query_us": 273, + "sequential_qps": 5024.33, + "sequential_pread_rounds": 112, + "sequential_pread_bytes": 113643808, + "batch_ms": 100, + "batch_qps": 9986.07, + "batch_pread_rounds": 2, + "batch_pread_bytes": 120397102 + }, + { + "dataset": "gist", + "implementation": "baseline", + "repeat": 1, + "n": 1000000, + "d": 960, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 6485, + "train_ms": 1829, + "add_ms": 3718, + "write_ms": 935, + "peak_rss_bytes": 5446975488, + "file_bytes": 973626471, + "recall_at_10": 0.8576, + "first_query_us": 4444, + "p50_query_us": 4296, + "p95_query_us": 4663, + "sequential_qps": 237.19, + "sequential_pread_rounds": 1892, + "sequential_pread_bytes": 74395897851, + "batch_ms": 1026, + "batch_qps": 974.58, + "batch_pread_rounds": 15, + "batch_pread_bytes": 961230121 + }, + { + "dataset": "gist", + "implementation": "optimized_cached", + "repeat": 1, + "n": 1000000, + "d": 960, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 5712, + "train_ms": 1696, + "add_ms": 3327, + "write_ms": 688, + "peak_rss_bytes": 5239685120, + "file_bytes": 973626471, + "recall_at_10": 0.94, + "first_query_us": 3771, + "p50_query_us": 1431, + "p95_query_us": 1788, + "sequential_qps": 687.92, + "sequential_pread_rounds": 162, + "sequential_pread_bytes": 891296688, + "batch_ms": 921, + "batch_qps": 1084.73, + "batch_pread_rounds": 15, + "batch_pread_bytes": 961230121 + }, + { + "dataset": "gist", + "implementation": "baseline", + "repeat": 2, + "n": 1000000, + "d": 960, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 6191, + "train_ms": 1727, + "add_ms": 3671, + "write_ms": 792, + "peak_rss_bytes": 5452349440, + "file_bytes": 973626471, + "recall_at_10": 0.8576, + "first_query_us": 4959, + "p50_query_us": 4060, + "p95_query_us": 4386, + "sequential_qps": 247.87, + "sequential_pread_rounds": 1892, + "sequential_pread_bytes": 74395897851, + "batch_ms": 1112, + "batch_qps": 899.2, + "batch_pread_rounds": 15, + "batch_pread_bytes": 961230121 + }, + { + "dataset": "gist", + "implementation": "optimized_cached", + "repeat": 2, + "n": 1000000, + "d": 960, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 5850, + "train_ms": 1701, + "add_ms": 3661, + "write_ms": 486, + "peak_rss_bytes": 5228003328, + "file_bytes": 973626471, + "recall_at_10": 0.94, + "first_query_us": 4340, + "p50_query_us": 1485, + "p95_query_us": 1828, + "sequential_qps": 665.58, + "sequential_pread_rounds": 162, + "sequential_pread_bytes": 891296688, + "batch_ms": 1007, + "batch_qps": 992.22, + "batch_pread_rounds": 15, + "batch_pread_bytes": 961230121 + }, + { + "dataset": "gist", + "implementation": "baseline", + "repeat": 3, + "n": 1000000, + "d": 960, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 6404, + "train_ms": 1741, + "add_ms": 3705, + "write_ms": 957, + "peak_rss_bytes": 5452398592, + "file_bytes": 973626471, + "recall_at_10": 0.8576, + "first_query_us": 4403, + "p50_query_us": 3970, + "p95_query_us": 4319, + "sequential_qps": 253.2, + "sequential_pread_rounds": 1892, + "sequential_pread_bytes": 74395897851, + "batch_ms": 1113, + "batch_qps": 897.67, + "batch_pread_rounds": 15, + "batch_pread_bytes": 961230121 + }, + { + "dataset": "gist", + "implementation": "optimized_cached", + "repeat": 3, + "n": 1000000, + "d": 960, + "train_n": 65536, + "nq": 1000, + "nlist": 1024, + "nprobe": 64, + "build_ms": 5866, + "train_ms": 1690, + "add_ms": 3613, + "write_ms": 563, + "peak_rss_bytes": 5227593728, + "file_bytes": 973626471, + "recall_at_10": 0.94, + "first_query_us": 4096, + "p50_query_us": 1466, + "p95_query_us": 1936, + "sequential_qps": 651.23, + "sequential_pread_rounds": 162, + "sequential_pread_bytes": 891296688, + "batch_ms": 1001, + "batch_qps": 998.4, + "batch_pread_rounds": 15, + "batch_pread_bytes": 961230121 + } + ], + "paimon_python": [ + { + "corpus": "sift", + "recall": 0.9812, + "p95_ms": 0.25953779331757676, + "qps": 4905.160454190915, + "batch_qps": 10024.995622783992, + "batch_s": [ + 0.09976404199551325, + 0.09975066699553281, + 0.09955183300189674 + ], + "repeat": 1 + }, + { + "corpus": "sift", + "recall": 0.9812, + "p95_ms": 0.26068329170811916, + "qps": 4853.263991636916, + "batch_qps": 9705.435943827024, + "batch_s": [ + 0.10371475000283681, + 0.10281208300148137, + 0.1030350419896422 + ], + "repeat": 2 + }, + { + "corpus": "sift", + "recall": 0.9812, + "p95_ms": 0.25635589117882773, + "qps": 4650.050805561705, + "batch_qps": 9926.987010163688, + "batch_s": [ + 0.10073550000379328, + 0.10022358300921042, + 0.10198312500142492 + ], + "repeat": 3 + }, + { + "corpus": "glove", + "recall": 0.876, + "p95_ms": 0.24610684049548576, + "qps": 5230.060561814994, + "batch_qps": 10923.012825206137, + "batch_s": [ + 0.09154983299958985, + 0.09192229199106805, + 0.09026929199171718 + ], + "repeat": 1 + }, + { + "corpus": "glove", + "recall": 0.876, + "p95_ms": 0.24004789374885144, + "qps": 5253.1115954928655, + "batch_qps": 11057.981564664737, + "batch_s": [ + 0.09043241699691862, + 0.09043366598780267, + 0.09005095799511764 + ], + "repeat": 2 + }, + { + "corpus": "glove", + "recall": 0.876, + "p95_ms": 0.23775805893819774, + "qps": 4362.8639692278075, + "batch_qps": 11051.921929271042, + "batch_s": [ + 0.09028120899165515, + 0.09048199999961071, + 0.10996950000117067 + ], + "repeat": 3 + }, + { + "corpus": "gist", + "recall": 0.9399, + "p95_ms": 1.6917080560233444, + "qps": 708.0206188151822, + "batch_qps": 1141.0011694863388, + "batch_s": [ + 0.8709349999990081, + 0.876423291003448, + 0.8788634160009678 + ], + "repeat": 1, + "batch_recall": 0.94 + }, + { + "corpus": "gist", + "recall": 0.9399, + "p95_ms": 1.7692583467578515, + "qps": 679.2866663514386, + "batch_qps": 981.6139216649766, + "batch_s": [ + 0.9849646670045331, + 1.0199584999936633, + 1.0187304580031196 + ], + "repeat": 2, + "batch_recall": 0.94 + }, + { + "corpus": "gist", + "recall": 0.9399, + "p95_ms": 1.8572527049400378, + "qps": 667.4037218128531, + "batch_qps": 986.4969530139211, + "batch_s": [ + 1.0136878750054166, + 1.0033612500119489, + 1.0273174589965492 + ], + "repeat": 3, + "batch_recall": 0.94 + } + ], + "lance": [ + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 0, + "n": 1000000, + "d": 128, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 0.31032995801069774, + "index_build_s": 3.6129824169911444, + "index_bytes": 131369309, + "recall": 0.9797, + "p50_ms": 1.0014379950007424, + "p95_ms": 1.2794169888366012, + "sequential_qps": 989.3228202686936, + "batch_s": [ + 0.2415808749938151, + 0.23636166600044817, + 0.2391535839997232 + ], + "batch_qps": 4181.413396678, + "batch_recall": 0.9797, + "dataset": "sift" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 0, + "n": 1000000, + "d": 128, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 0.31032995801069774, + "index_build_s": 3.6129824169911444, + "index_bytes": 131369309, + "recall": 0.9793, + "p50_ms": 0.8949789917096496, + "p95_ms": 1.0652399017999412, + "sequential_qps": 1102.8434043752866, + "batch_s": [ + 0.3082639579952229, + 0.31552245799684897, + 0.30906400000094436 + ], + "batch_qps": 3235.5758030600277, + "batch_recall": 0.9793, + "dataset": "sift" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 1, + "n": 1000000, + "d": 128, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 0.5087806249939604, + "index_build_s": 3.1083727079967503, + "index_bytes": 131369053, + "recall": 0.9771, + "p50_ms": 1.2891464939457364, + "p95_ms": 1.631252094375668, + "sequential_qps": 844.603676273628, + "batch_s": [ + 0.2487101669976255, + 0.25258970899449196, + 0.2553312910022214 + ], + "batch_qps": 3958.9894773654705, + "batch_recall": 0.9771, + "dataset": "sift" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 1, + "n": 1000000, + "d": 128, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 0.5087806249939604, + "index_build_s": 3.1083727079967503, + "index_bytes": 131369053, + "recall": 0.9772, + "p50_ms": 1.0143955005332828, + "p95_ms": 5.934893446101337, + "sequential_qps": 495.67022593381444, + "batch_s": [ + 0.38212937500793487, + 0.5658738329948392, + 0.46950991699122824 + ], + "batch_qps": 2129.8804643112207, + "batch_recall": 0.9772, + "dataset": "sift" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 2, + "n": 1000000, + "d": 128, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 0.5426220420049503, + "index_build_s": 6.131838665998657, + "index_bytes": 131368925, + "recall": 0.9775, + "p50_ms": 1.119521002692636, + "p95_ms": 3.3385020491550677, + "sequential_qps": 591.3198932006269, + "batch_s": [ + 0.22399091700208373, + 0.21637758301221766, + 0.21978304100048263 + ], + "batch_qps": 4549.941594437235, + "batch_recall": 0.9775, + "dataset": "sift" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 2, + "n": 1000000, + "d": 128, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 0.5426220420049503, + "index_build_s": 6.131838665998657, + "index_bytes": 131368925, + "recall": 0.977, + "p50_ms": 0.9637295006541535, + "p95_ms": 1.0986836452502757, + "sequential_qps": 1017.894235910497, + "batch_s": [ + 0.27645879100600723, + 0.2729735000029905, + 0.26712979200237896 + ], + "batch_qps": 3663.3592637711895, + "batch_recall": 0.977, + "dataset": "sift" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 0, + "n": 1183514, + "d": 100, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 0.4582100830011768, + "index_build_s": 2.8858480420021806, + "index_bytes": 123788044, + "recall": 0.784, + "p50_ms": 1.1995829991064966, + "p95_ms": 1.499956246698275, + "sequential_qps": 826.7088574803406, + "batch_s": [ + 0.23057912499643862, + 0.23458475001098122, + 0.23765391600318253 + ], + "batch_qps": 4262.851698386995, + "batch_recall": 0.784, + "dataset": "glove" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 0, + "n": 1183514, + "d": 100, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 0.4582100830011768, + "index_build_s": 2.8858480420021806, + "index_bytes": 123788044, + "recall": 0.7841, + "p50_ms": 0.9254164979211055, + "p95_ms": 1.0633017438522072, + "sequential_qps": 1069.245148811762, + "batch_s": [ + 0.3005618330062134, + 0.29484195800614543, + 0.28533616701315623 + ], + "batch_qps": 3391.6475347079227, + "batch_recall": 0.7841, + "dataset": "glove" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 1, + "n": 1183514, + "d": 100, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 0.42901387499296106, + "index_build_s": 2.789812582996092, + "index_bytes": 123794125, + "recall": 0.7845, + "p50_ms": 1.251395995495841, + "p95_ms": 1.5928913599054795, + "sequential_qps": 796.3681045469514, + "batch_s": [ + 0.23582358300336637, + 0.23455374999321066, + 0.24161516700405627 + ], + "batch_qps": 4240.458003666771, + "batch_recall": 0.7845, + "dataset": "glove" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 1, + "n": 1183514, + "d": 100, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 0.42901387499296106, + "index_build_s": 2.789812582996092, + "index_bytes": 123794125, + "recall": 0.7845, + "p50_ms": 0.9430415011593141, + "p95_ms": 1.0890444049437062, + "sequential_qps": 1050.3114299475167, + "batch_s": [ + 0.32420116700814106, + 0.2876453750068322, + 0.30344629200408235 + ], + "batch_qps": 3295.476090334123, + "batch_recall": 0.7845, + "dataset": "glove" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 2, + "n": 1183514, + "d": 100, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 0.35886495900922455, + "index_build_s": 2.755436583989649, + "index_bytes": 123799052, + "recall": 0.7845, + "p50_ms": 1.228708504640963, + "p95_ms": 1.5429267579747827, + "sequential_qps": 804.5836861276184, + "batch_s": [ + 0.23650141699181404, + 0.24138320799102075, + 0.2385975840006722 + ], + "batch_qps": 4191.157275076107, + "batch_recall": 0.7845, + "dataset": "glove" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 2, + "n": 1183514, + "d": 100, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 0.35886495900922455, + "index_build_s": 2.755436583989649, + "index_bytes": 123799052, + "recall": 0.7843, + "p50_ms": 1.055813001585193, + "p95_ms": 1.403788961761165, + "sequential_qps": 872.8329078474785, + "batch_s": [ + 0.30117095899186097, + 0.29349999999976717, + 0.3128447079943726 + ], + "batch_qps": 3320.373263568964, + "batch_recall": 0.7843, + "dataset": "glove" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 0, + "n": 1000000, + "d": 960, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 4.566240332991583, + "index_build_s": 18.651229708993924, + "index_bytes": 966467340, + "recall": 0.9249, + "p50_ms": 3.5183125000912696, + "p95_ms": 4.213342302682577, + "sequential_qps": 279.86572814988415, + "batch_s": [ + 0.7250397089956095, + 0.7115559579979163, + 0.7171322499925736 + ], + "batch_qps": 1394.4429357490974, + "batch_recall": 0.9249, + "dataset": "gist" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 0, + "n": 1000000, + "d": 960, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 4.566240332991583, + "index_build_s": 18.651229708993924, + "index_bytes": 966467340, + "recall": 0.9248, + "p50_ms": 1.60639600653667, + "p95_ms": 2.045174988597864, + "sequential_qps": 605.6153423224837, + "batch_s": [ + 0.7599140410020482, + 0.7975927080115071, + 0.7592186659894651 + ], + "batch_qps": 1315.9383114981879, + "batch_recall": 0.9248, + "dataset": "gist" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 1, + "n": 1000000, + "d": 960, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 3.40062129200669, + "index_build_s": 18.463206583997817, + "index_bytes": 966466660, + "recall": 0.9326, + "p50_ms": 3.4582089938339777, + "p95_ms": 4.417935459787259, + "sequential_qps": 282.8399689006948, + "batch_s": [ + 0.7494371249922551, + 0.7551345410029171, + 0.7434802909992868 + ], + "batch_qps": 1334.3347515781718, + "batch_recall": 0.9326, + "dataset": "gist" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 1, + "n": 1000000, + "d": 960, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 3.40062129200669, + "index_build_s": 18.463206583997817, + "index_bytes": 966466660, + "recall": 0.9325, + "p50_ms": 1.558103998831939, + "p95_ms": 1.8234190487419255, + "sequential_qps": 638.5420358786371, + "batch_s": [ + 0.8040630000032252, + 0.8117980000097305, + 0.8028900000063004 + ], + "batch_qps": 1243.6836416997037, + "batch_recall": 0.9325, + "dataset": "gist" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 2, + "n": 1000000, + "d": 960, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 0, + "data_write_s": 2.69620574999135, + "index_build_s": 19.185824375002994, + "index_bytes": 966467596, + "recall": 0.9249, + "p50_ms": 3.6111040026298724, + "p95_ms": 4.190530948108062, + "sequential_qps": 277.0624360371063, + "batch_s": [ + 0.7597572090016911, + 0.772940082999412, + 0.7539892079948913 + ], + "batch_qps": 1316.2099525373167, + "batch_recall": 0.9249, + "dataset": "gist" + }, + { + "engine": "lance", + "version": "11.0.0", + "machine": "arm64", + "repeat": 2, + "n": 1000000, + "d": 960, + "nq": 1000, + "k": 10, + "nlist": 1024, + "train_n": 65536, + "threads": 8, + "nprobe": 64, + "query_parallelism": 8, + "data_write_s": 2.69620574999135, + "index_build_s": 19.185824375002994, + "index_bytes": 966467596, + "recall": 0.9249, + "p50_ms": 1.60087500262307, + "p95_ms": 1.8618172958667856, + "sequential_qps": 621.3550108671919, + "batch_s": [ + 0.8047680419986136, + 0.8261880420031957, + 0.8697780829970725 + ], + "batch_qps": 1210.378205880801, + "batch_recall": 0.9249, + "dataset": "gist" + } + ], + "hardware": { + "cpu": "Apple M4 Pro", + "logical_cpus": 12, + "ram_bytes": 51539607552 + }, + "dataset_sources": { + "gist": { + "url": "https://ann-benchmarks.com/gist-960-euclidean.hdf5", + "content_length": 3844648288, + "etag": "34da1d8a80764582ee4b0c0839b7c32a-459", + "n": 1000000, + "d": 960, + "query_count": 1000, + "normalized": false + } + } +} diff --git a/docs/development.html b/docs/development.html index 6993a82..fae82fc 100644 --- a/docs/development.html +++ b/docs/development.html @@ -26,7 +26,7 @@

Build · test · measure

Development and benchmarks

Run the standard Rust checks, exercise the C/C++, Java, and Python integrations, and measure ANN and filtered-query behavior with reproducible workloads.

Cargo workspaceCMake smoke testsMaven / pytestCriterion benchmarks
- +

Repository layout

@@ -103,7 +103,32 @@

Public SIFT1M, GIST1M, and GloVe-100 data

Physical versus intrinsic dimensionANN_NOISE_DIMENSIONS=64 spreads independent per-vector noise evenly over 64 of the 1024 stored dimensions; the other dimensions retain cluster-center signal. This creates a repeatable storage/compute scale workload without pretending that 1024 independent uniform-noise dimensions resemble production embeddings. With all 1024 dimensions independently noisy, same-cluster distances concentrate so strongly that exact Top-10 membership is nearly arbitrary: a 65k diagnostic produced local Recall@10 of 0.0125 at l_search=200 and only 0.178 at l_search=5000. Use a public or production corpus for an algorithm-quality claim.
VariableMeaning
ANN_DATASET_NAMEComma-free label written to every CSV row
ANN_BASE_FVECS / ANN_QUERY_FVECS / ANN_GROUND_TRUTH_IVECSOptional public-data files; set all three together. Their vector count, query count, dimension, and ground-truth width are inspected before the full payload is loaded.
ANN_N / ANN_NQ / ANN_DInferred for public files; an explicit value is validated as a shape assertion. Generated workloads retain defaults of 20,000 / 64 / 64.
ANN_DATA_GIBGenerated-workload raw float-vector GiB; derives n = floor(bytes / (4d)) and cannot be combined with public file paths.
ANN_TRAIN_NOptional training-sample override; omitted values resolve to min(N, max(65,536, 64 × nlist)).
ANN_KTop K
ANN_INDEXESall or a comma-separated subset of the five index names. Public multi-index runs automatically isolate each index in a child process.
ANN_STORAGE_CASESall or a comma-separated subset of local_ssd_warm_cache, remote_cache_2ms, and object_store_20ms. The default runs all three.
ANN_NLIST / ANN_NPROBEIVF list count / probed lists
ANN_PQ_CODE_RATIOIVF-PQ and DiskANN PQ-code/raw-vector byte ratio; default 0.0625 and automatically resolves a valid m
ANN_PQ_MOptional explicit subquantizer-count override; takes precedence over automatic sizing
ANN_RQ_BITSPersisted IVF-RQ bit width, 1–8; default 4
ANN_DISKANN_L_SEARCH / ANN_DISKANN_L_SEARCHESOne DiskANN search-list size, or a comma-separated sweep over one built index
ANN_DISKANN_BUILD_DISTANCEproduct_quantized by default; use full_precision as a graph-quality control
ANN_DISKANN_RAW_VECTOR_ENCODINGf16 by default; use f32 as an exact-distance and numeric-range control. The resolved value is written to every CSV row.
ANN_DISKANN_MEMORY_BUDGET_BYTESDiskANN's internal preflight build budget; default 8 GiB and excludes the benchmark's source-vector allocation
ANN_NOISE_DIMENSIONS / ANN_CLUSTERS / ANN_SEEDGenerated-workload distribution and reproducibility controls; ignored by public file loading.
ANN_OUTPUT_DIRDirectory whose filesystem supplies the local-storage path
ANN_KEEP_INDEXES1 retains generated index files; the default deletes each index after all selected serving cases
ANN_REUSE_INDEX_PATHSkips training/build and queries one retained index; requires a single ANN_INDEXES value
Large-run memory boundaryThe current public writer API accepts an in-memory vector slice and each index writer retains its encoded or raw build state. A raw 10 GiB DiskANN run therefore needs roughly the 10 GiB source allocation plus DiskANN's separately budgeted build state. On a 48 GiB machine, use a 32 GiB DiskANN budget and run one index per process. This benchmark validates large immutable builds; it is not a streaming-ingest benchmark.
-

The benchmark executes three serving cases by default. local_ssd_warm_cache uses the real index file with no artificial delay. remote_cache_2ms adds a fixed 2 ms delay per pread call while reading all ranges in that call concurrently. object_store_20ms uses the ObjectStore read plan: open/optimization and sequential-query latency are the measured CPU/I/O time plus 20 ms for every observed dependent read round, avoiding hours of idle wall-clock sleep while retaining the exact fixed-latency model; batch QPS still injects the real 20 ms sleep so concurrent-client overlap is measured rather than inferred. These models do not include bandwidth or operating-system cache misses. DiskANN uses a dedicated range-I/O pool separate from Rayon query workers, preventing a saturated batch from starving nested positional reads; IVF retains the shared compute pool because its batch read scheduling is not nested inside per-query sessions. The unified IVF Reader reuses the 64-byte type-dispatch header and loads resident metadata in one further contiguous read, reducing initialization from three rounds to two without changing any format. DiskANN is opened explicitly as LocalStorage, RemoteStorage, or ObjectStore with the default 4 GiB Reader budget, which is automatically partitioned into resident state and caches. The sequential sweep and batch call use separate opened-and-optimized Readers, preventing all benchmark queries from prewarming their own batch measurement. CSV output records optimization rounds, ranges, and bytes separately from query I/O. Use ANN_STORAGE_CASES for a focused rerun against a retained index. See the recorded public-corpus results and interpretation.

+

The benchmark executes three serving cases by default. local_ssd_warm_cache uses the real index file with no artificial delay. remote_cache_2ms adds a fixed 2 ms delay per pread call while reading all ranges in that call concurrently. object_store_20ms uses the ObjectStore read plan: open/optimization and sequential-query latency are the measured CPU/I/O time plus 20 ms for every observed dependent read round, avoiding hours of idle wall-clock sleep while retaining the exact fixed-latency model; batch QPS still injects the real 20 ms sleep so concurrent-client overlap is measured rather than inferred. These models do not include bandwidth or operating-system cache misses. DiskANN uses a dedicated range-I/O pool separate from Rayon query workers, preventing a saturated batch from starving nested positional reads; IVF retains the shared compute pool because its batch read scheduling is not nested inside per-query sessions. The unified IVF Reader reuses the 64-byte type-dispatch header and loads resident metadata in one further contiguous read, reducing initialization from three rounds to two without changing any format. DiskANN is opened explicitly as LocalStorage, RemoteStorage, or ObjectStore with the default 4 GiB Reader budget, which is automatically partitioned into resident state and caches. The sequential sweep and batch call use separate opened-and-optimized Readers, preventing all benchmark queries from prewarming their own batch measurement. IVF-SQ now uses the default reader budget for a partition cache: sequential queries can reuse prior partitions, while the first batch includes payload reads and cache insertion. Simulated remote delay applies only to actual positional-read calls; cache hits issue none. CSV output records optimization rounds, ranges, and bytes separately from query I/O. Use ANN_STORAGE_CASES for a focused rerun against a retained index. See the recorded public-corpus results and interpretation.

+
+ +
+

Reproduce the IVF-SQ / Lance comparison

+

The September 2026 results compare SIFT1M, GIST1M, and normalized GloVe-100 with Lance 11.0.0. Use the same converted public base, query, and ground-truth files for both engines. Install numpy, h5py, pyarrow, and pylance==11.0.0 in a temporary Python environment. Run from the repository root, one benchmark at a time, with no concurrent compilation.

+
    +
  1. Convert the public data using the commands above, retaining the first 1,000 held-out queries. Use --normalize-l2 for GloVe before either engine reads it.
    2. +
  2. Build with cargo bench -p paimon-vindex-core --bench ann_bench --no-run and cargo build --release -p paimon-vindex-ffi. For Paimon, set ANN_INDEXES=IVF_SQ, RAYON_NUM_THREADS=8, ANN_TRAIN_N=65536, ANN_NLIST=1024, ANN_NPROBE=64, ANN_K=10, and ANN_STORAGE_CASES=local_ssd_warm_cache. Set the three public-data paths, ANN_KEEP_INDEXES=1, and an ANN_OUTPUT_DIR; run three separate build repetitions.
    3. +
  3. Run benchmark_lance_ivfsq.py on the same inputs. It reports source-dataset writing separately from index construction, tests both query scheduling modes, and warms the probed partitions before timing.
    4. +
  4. Run benchmark_ivfsq_reader.py on a retained ivf_sq.index. It uses the public Python reader with ordinary positional file-read callbacks and warms partitions through batch search before timing.
    5. +
+
Shell · after exporting the public ANN paths and setting OUT and INDEX
python tools/benchmark_lance_ivfsq.py \
+  --base "$ANN_BASE_FVECS" --queries "$ANN_QUERY_FVECS" \
+  --ground-truth "$ANN_GROUND_TRUTH_IVECS" --output-dir "$OUT/lance" \
+  --threads 8 --train-n 65536 --nlist 1024 --nprobe 64 --nq 1000 --k 10 \
+  --query-parallelism 0 8 --cache-bytes 1073741824 --repeats 3
+
+PYTHONPATH=python PAIMON_VINDEX_LIB_PATH="$PWD/target/release/libpaimon_vindex_ffi.dylib" \
+python tools/benchmark_ivfsq_reader.py --index "$INDEX" \
+  --queries "$ANN_QUERY_FVECS" --ground-truth "$ANN_GROUND_TRUTH_IVECS" \
+  --threads 8 --nprobe 64 --nq 1000 --k 10 \
+  --memory-budget-bytes 4294967296 --repeats 3
+

OUT is the benchmark output directory and INDEX is the file retained under ANN_OUTPUT_DIR/<pid>/ivf_sq.index. Use the .so library on Linux. The full command sequence includes data conversion and Paimon build settings; raw JSON preserves all repetitions.

+
Keep the measurement boundaries explicitThe report's query table uses warmed Python readers for both engines; Rust ann_bench times its batch on a fresh reader, including reads and cache insertion. IVF-SQ's warmup_queries does not load partitions: use actual searches. A reader budget of zero measures the uncached SQ path, not a cold operating-system cache.
+

Paimon's budget is 4 GiB and Lance's index cache is 1 GiB; both fit each measured index. Lance uses the byte-count option index_cache_size_bytes. Its deprecated index_cache_size counts entries and must not be used as a byte budget. Match training sample counts while allowing each engine its native training algorithm and iteration policy. Report recall alongside latency, exclude Lance source-dataset writing from index-build comparisons, and state which scheduling mode supplies each metric.

diff --git a/docs/index.html b/docs/index.html index 685e09b..58dc7f3 100644 --- a/docs/index.html +++ b/docs/index.html @@ -9,7 +9,7 @@ - + Index Selection Guide · Paimon Vector Index @@ -57,7 +57,7 @@

Five indexes.
One selection map.

@@ -108,8 +108,8 @@

Core differences

IndexRepresentationCandidate searchMain payload per vectorAccuracy profileBuild costPrimary controlsBest fit IVF-PQ8-bit PQ codes; optional OPQDistance-table lookup over compact codesAbout m bytesPQ reconstruction error; OPQ may improve uneven subspacesMedium to highAutomatic nlist/nprobe/pq.m; target-based OPQ; explicit overridesMinimum file and selected-list bytes among IVF when the measured 0.58–0.74 recall band is sufficient - IVF-SQ8-bit scalar residual codes with per-list boundsSIMD code scan in probed listsAbout d bytesPer-coordinate scalar quantization lossLowAutomatic nlist/nprobe; explicit overridesHighest measured compact batch throughput when 0.80–0.86 recall meets the target - IVF-RQMulti-bit rotated residual levels + coarse/full factorsBounded sign-plane scan, then full bit-plane refinementDefault about padded_d/2+20 bytesMeasured 0.82–0.91 Recall@10 across GloVe-100, SIFT1M, and GIST1M at four bitsLow to mediumAutomatic nlist/nprobe; budget-based bits; explicit overridesBest measured compact recall when 3–4.5× lower batch throughput than IVF-SQ is acceptable + IVF-SQ8-bit scalar residual codes with pooled training boundsSIMD code scan in probed listsAbout d bytesPer-coordinate scalar quantization lossLowAutomatic nlist/nprobe; explicit overridesCompact SIMD scans with pooled training bounds and a bounded partition cache + IVF-RQMulti-bit rotated residual levels + coarse/full factorsBounded sign-plane scan, then full bit-plane refinementDefault about padded_d/2+20 bytesMeasured 0.82–0.91 Recall@10 across GloVe-100, SIFT1M, and GIST1M at four bitsLow to mediumAutomatic nlist/nprobe; budget-based bits; explicit overridesConfigurable multi-bit compression when SQ codes are too large; measure recall and scan cost at the chosen width DiskANNGlobal Vamana + resident PQ + persisted rerank vectorsPQ-guided graph traversal and F32/F16 rerankE·d + pq.m + 4(R+1) bytes, approximately; E=4 or 2Approximate candidate discovery; F32-exact or F16-quantized distances for reranked candidatesVery highBuild preset + deployment/capacity objectives; calibrated automatic l_searchImmutable L2 on local SSD when high recall and sub-MiB query reads repay the long build IVF-FLATRaw f32 vectorsExact distance scan in probed listsAbout 4d bytesNo quantization loss; recall mainly depends on nprobeLowAutomatic nlist/nprobe; explicit overridesRecall ceiling, frequent rebuilds, IP/cosine, or production sets whose scan bytes are affordable @@ -117,12 +117,36 @@

Core differences

+
+
+

6 September 2026: IVF-SQ versus Lance

+

Three-run medians on public SIFT1M, GIST1M, and GloVe-100: Apple M4 Pro, eight workers, 1,024 partitions, 64 probes, Top-10, 65,536 training rows, and 1,000 held-out queries. Both engines use the same input files and native training defaults; GloVe is normalized before either engine reads it.

+
+
+ + + + + + +
CorpusEngineIndex buildPython P95Python batch QPSRecall@10
SIFT1MPaimon IVF-SQ0.886 s0.260 ms9,9270.9812
SIFT1MLance 11.0.03.613 s1.099 ms4,1810.9772–0.9775
GIST1MPaimon IVF-SQ5.850 s1.769 ms9860.9399
GIST1MLance 11.0.018.651 s1.862 ms1,3340.9249
GloVe-100Paimon IVF-SQ0.797 s0.240 ms11,0520.8760
GloVe-100Lance 11.0.02.790 s1.089 ms4,2400.7843–0.7845
+

Build includes training through index serialization and excludes Lance source-dataset writing. Queries use both public Python interfaces after warming selected partitions, with no raw-vector refinement. Cache budgets are 4 GiB for Paimon and 1 GiB for Lance; each measured index fits both.

+

Relative to Lance, SIFT1M: 4.08× build speed, 76% lower Python P95, 2.37× batch throughput; GIST1M: 3.19× build speed, 5% lower Python P95, 0.74× batch throughput; GloVe-100: 3.50× build speed, 78% lower Python P95, 2.61× batch throughput.

+
GIST batch throughput still trails LanceIVF-SQ reaches 986 QPS versus Lance's 1,334 QPS, about 26% lower, despite its faster build and higher recall. Its median Python P95 is only 5% lower, and the three-run latency ranges overlap. These results do not establish an across-the-board performance lead.
+

Lance uses its faster scheduling mode separately for each metric: SIFT1M P95 / batch parallelism 8 / 0; GIST1M P95 / batch parallelism 8 / 0; GloVe-100 P95 / batch parallelism 8 / 0. Recall ranges show the two scheduling-mode medians. These measurements do not cover cold storage, object stores, or other architectures.

+

See the full report, reproduction commands, and all recorded runs. The native build and local-storage tables below also contain the refreshed IVF-SQ results.

+
+
-

Measured comparison: public SIFT1M, GIST1M, and GloVe-100 corpora

-

This repository's unified benchmark builds all five indexes over standard public vectors, searches the same independent public queries, and scores every result against published exact neighbors. The results below are the homepage's sole performance evidence.

+

Public-corpus benchmark results

+

IVF-SQ was refreshed on 6 September 2026 across all three corpora. Its build and local-search cells below use three-run native ann_bench medians with eight Rayon workers. The other four index rows retain their July 2026 measurements with 12 workers; these tables record the latest available results for each index, not a new simultaneous five-index ranking.

+

All rows use the same public corpus shapes, 1,000 independent queries, nlist=1024, nprobe=64, and Top-10. Native batch timing includes the first payload reads and cache insertion on a fresh reader, so it differs from the warmed Python comparison above. The remote-model results remain archived separately below.

+
+ Historical five-index setup and reproduction parameters · July 2026 +

The following records the original 12-worker comparison. For the refreshed eight-worker IVF-SQ configuration, use the current reproduction guide.

Benchmark setup

This is not the zero-configuration benchmarkRunning cargo bench -p paimon-vindex-core --bench ann_bench without public file paths uses a 20k-vector, 64-dimensional generated smoke workload. Reproducing the results below requires the public files, recorded IVF and DiskANN search settings, a fixed worker count, and an output directory on the storage device being measured. File shape, training count, and multi-index process isolation are automatic.

The real-data run uses the public ANN-Benchmarks SIFT1M, GIST1M, and GloVe-100 files, the first 1,000 independent test queries, and their published Top-100 exact neighbors. Recall@10 compares only the first ten published neighbors. SIFT and GIST contain one million base vectors with 128 and 960 dimensions. GloVe contains 1,183,514 vectors with 100 dimensions and angular ground truth; its base and query vectors are L2-normalized during conversion so the common L2 benchmark produces the same neighbor ordering as cosine. The benchmark supplies 65,536 base vectors to every trainer; DiskANN bounds PQ training memory with a deterministic reservoir of at most 50,000 vectors.

@@ -140,14 +164,30 @@

Benchmark setup

Equal relative PQ budgetThe default pq.code-ratio=0.0625 automatically resolves SIFT to pq.m=32, GIST to pq.m=240, and GloVe to pq.m=25. Every code occupies 6.25% as many bytes as its raw f32 vector and leaves four dimensions per PQ sub-vector. The concrete value is persisted in index metadata; use explicit pq.m only as an expert override.

The cross-index run was recorded on 25 July 2026 using an Apple M4 Pro with 12 logical CPUs and 48 GiB RAM, a release build with Rust 1.95, real APFS files with warm operating-system pages, and the automatic 4 GiB DiskANN Reader budget. The IVF-RQ staged A/B and rebased IVF-PQ warm-local refresh were recorded on 30 July on the same host and toolchain. The reproduction command pins Rayon to 12 workers instead of relying on automatic host parallelism. The modeled serving profiles add 2 ms or 20 ms per positional-read round while executing all ranges in that round concurrently. DiskANN's benchmark adapter runs those ranges on a separate 12-worker I/O pool so a full query-worker pool cannot starve nested reads; this models the independent executor required of a production concurrent storage callback. For the 20 ms profile, open/optimization and sequential-query latency are computed as measured CPU/I/O time plus 20 ms per observed round; batch QPS retains literal delay injection so query overlap is measured. IVF multi-range calls are bounded to 64 MiB, so an all-query GIST batch uses 4 IVF-PQ, 15 IVF-SQ, or 59 IVF-FLAT payload rounds instead of submitting hundreds of MiB or several GiB as one unbounded call. Unified IVF Readers now reuse the 64-byte dispatch header, so opening and loading resident metadata takes two positional-read rounds rather than three. Each dataset's three DiskANN profile rows reuse the same built graph. Sequential and batch measurements use separately opened and optimized Readers, so the batch does not inherit query-dependent windows from the sequential sweep. Batch QPS measures one search_batch call over all 1,000 public queries; it is not concurrent-client QPS. See the complete public-data command.

-

Build, file, and peak process memory

+
+ +

Build, file, and peak process memory

- +
IndexSIFT buildSIFT file / RSSGIST buildGIST file / RSSGloVe buildGloVe file / RSS
IVF-PQ8.74 s0.032 / 0.88 GiB55.4 s0.230 / 5.00 GiB7.92 s0.030 / 0.85 GiB
IVF-SQ3.93 s0.122 / 0.79 GiB22.7 s0.907 / 5.09 GiB3.86 s0.113 / 0.71 GiB
IVF-SQ · 6 Sep0.886 s0.122 / 0.77 GiB5.850 s0.907 / 4.87 GiB0.797 s0.113 / 0.72 GiB
IVF-RQ3.92 s0.080 / 0.71 GiB23.5 s0.471 / 4.65 GiB4.03 s0.095 / 0.68 GiB
DiskANN74.0 s0.361 / 1.51 GiB11 min 26 s2.089 / 7.94 GiB2 min 33 s0.396 / 1.45 GiB
IVF-FLAT4.05 s0.479 / 1.83 GiB24.9 s3.582 / 12.74 GiB4.10 s0.443 / 1.60 GiB
+

Native local-storage results

+
+ + + + + +
Index / searchSIFT Recall / P95 / batch QPS / readGIST Recall / P95 / batch QPS / readGloVe Recall / P95 / batch QPS / read
IVF-PQ0.7142 / 0.72 ms / 7,899 / 2.18 MiB0.7410 / 2.47 ms / 950 / 17.84 MiB0.5819 / 0.64 ms / 8,048 / 1.84 MiB
IVF-SQ · 6 Sep0.9811 / 0.298 ms / 8,987 / 0.115 MiB0.9400 / 1.828 ms / 998 / 0.850 MiB0.8760 / 0.282 ms / 10,009 / 0.108 MiB
IVF-RQ0.9148 / 1.20 ms / 3,074 / 5.54 MiB0.9039 / 4.41 ms / 444 / 37.02 MiB0.8203 / 1.23 ms / 2,917 / 5.89 MiB
DiskANN0.9915 / 1.50 ms / 9,009 / 0.66 MiB0.9336 / 1.83 ms / 4,651 / 0.83 MiB0.8355 / 1.90 ms / 6,289 / 0.96 MiB
IVF-FLAT0.9937 / 1.88 ms / 8,510 / 33.19 MiB0.9549 / 11.38 ms / 875 / 283.40 MiB0.8832 / 1.40 ms / 9,502 / 27.57 MiB
+

The refreshed SQ reader reuses decoded partitions within its memory budget. The read column is average payload I/O per sequential query, including cache misses while that sweep warms the reader; it is not the full selected-list size or a cold-cache promise. SQ batch QPS includes the first batch's payload reads on a separate reader. The recorded July rows predate this SQ cache and training change.

+

For IVF-SQ, train / encode-add / serialize medians in SIFT / GIST / GloVe order are 252 / 538 / 93 ms; 1696 / 3613 / 563 ms; 187 / 517 / 90 ms. Stage medians are computed independently and need not sum to the median total. See the same-configuration baseline comparison for the optimization's measured effect.

+ +
+ Historical implementation notes and earlier SQ scores · July 2026 +

These notes preserve earlier snapshots. Their performance and implementation descriptions have been superseded where the refreshed tables above provide results.

IVF-SQ build and scan refreshThe add path now borrows L2/IP input, assigns rows once, and encodes lists in parallel with one residual scratch vector per active list task instead of materializing an additional N × d residual matrix. In the immediately preceding same-machine run, SIFT/GIST/GloVe peak RSS was 1.81 / 12.92 / 1.60 GiB; it is now 0.79 / 5.09 / 0.71 GiB. A Top-K threshold fast path skips hash work for candidates that cannot enter the heap: local P95 is now 0.79 / 3.56 / 0.71 ms and batch throughput is 11,082 / 1,502 / 12,962 QPS. An experimental list-major batch scan was slower on SIFT/GIST and was not retained. The blocked-code format, file size, read bytes, and measured Recall@10 remain unchanged.
30 July IVF-PQ batch-table reuse refreshThe rebased Reader retains the v1 zero-copy/transposed-code, ordered-list, one-byte row-ID, and first-column accumulation fast paths. For large 8-bit residual-L2 batches, the default Auto mode now factors each distance table into reusable per-list and per-query components when the reuse heuristic and 64 MiB working-memory guard both pass; small or unsuitable batches keep the direct path, and callers may explicitly select On or Off. Six same-file runs per mode alternated execution order. SIFT/GIST/GloVe median batch throughput changed from 4,191 / 497 / 4,366 QPS with reuse disabled to 7,899 / 950 / 8,048 QPS with Auto, gains of 88.5% / 91.3% / 84.3%. The Auto medians used below are 0.72 / 2.47 / 0.64 ms P95 and 1,583 / 467 / 1,802 sequential QPS. File bytes, query bytes, and the v1 format are unchanged. GIST and GloVe Recall@10 are unchanged at four decimals; SIFT moved from 0.7143 to 0.7142 because the stable f64 factored-table path is numerically close but not bit-identical to direct residual-table accumulation. A removed contiguous all-query table remains distinct from this bounded factorization. Faiss FastScan's 4-bit, 32-row design remains a different accuracy/format choice.
Latest IVF-FLAT storage and scan reviewThe v1 writer retains only sort permutations and encoded IDs, materializes one sorted raw-vector list at a time, and reproduced all three prior public files byte-for-byte. The Reader now receives list bytes directly into an f32-aligned allocation; an internal prefix of at most three bytes keeps the raw-vector suffix aligned despite variable-length row IDs, so search no longer allocates and decodes a second vector payload. Together with the strict partial-L2 cutoff, the complete public rerun changed SIFT/GIST/GloVe local batch throughput from 6,570 / 559 / 6,345 to 8,510 / 875 / 9,502 QPS and P95 from 5.31 / 47.04 / 4.76 ms to 1.88 / 11.38 / 1.40 ms. Recall@10, file version, file bytes, and bytes read are unchanged; these are complete-run results, not best-of measurements.
@@ -169,16 +209,11 @@

Build, file, and peak process memory

Final open-source cross-check and format decisionFaiss FastScan still trades down to 4-bit lookup tables and a 32-row layout, so it is not a transparent replacement for the published 8-bit IVF-PQ v1. Faiss Panorama's additional level-oriented energy data was not needed to keep the existing IVF-FLAT v1 progressive cutoff. Lance's partition prefetch and transposed PQ storage match the current batched Readers; its prepared transposed L2 target is explicitly aimed at small target sets such as PQ codebooks, not large flat lists. Faiss RaBitQ's blocked multi-bit scan remains structurally aligned with IVF-RQ, while a direct-factor RQ payload experiment regressed SIFT batch throughput by 10–15% and was removed. DiskANN3's asynchronous provider, beam, and working-set model remains aligned with SeekRead, the latency-derived read planner, and the bounded caches. No measured result justified a v2 migration for IVF-PQ or IVF-FLAT, and no byte-layout change was retained for the pre-release IVF-SQ, IVF-RQ, or DiskANN formats.

DiskANN spends almost all build time constructing one global graph: about 18× IVF-FLAT on SIFT, 28× on GIST, and 37× on GloVe. The balanced F16 default makes its files about 42% smaller than IVF-FLAT on GIST and 11% smaller on GloVe, but they remain much larger than the compact IVF encodings because persisted rerank vectors, resident codes, and graph edges are all material. Peak RSS remains below the raw IVF writers because the DiskANN writer does not retain a second full raw-vector organization.

-

Warm local-storage result

-
- - - - - -
Index / searchSIFT Recall / P95 / batch QPS / readGIST Recall / P95 / batch QPS / readGloVe Recall / P95 / batch QPS / read
IVF-PQ0.7142 / 0.72 ms / 7,899 / 2.18 MiB0.7410 / 2.47 ms / 950 / 17.84 MiB0.5819 / 0.64 ms / 8,048 / 1.84 MiB
IVF-SQ0.8627 / 0.79 ms / 11,082 / 8.38 MiB0.8577 / 3.56 ms / 1,502 / 70.95 MiB0.8036 / 0.71 ms / 12,962 / 6.99 MiB
IVF-RQ0.9148 / 1.20 ms / 3,074 / 5.54 MiB0.9039 / 4.41 ms / 444 / 37.02 MiB0.8203 / 1.23 ms / 2,917 / 5.89 MiB
DiskANN0.9915 / 1.50 ms / 9,009 / 0.66 MiB0.9336 / 1.83 ms / 4,651 / 0.83 MiB0.8355 / 1.90 ms / 6,289 / 0.96 MiB
IVF-FLAT0.9937 / 1.88 ms / 8,510 / 33.19 MiB0.9549 / 11.38 ms / 875 / 283.40 MiB0.8832 / 1.40 ms / 9,502 / 27.57 MiB
-

IVF-SQ is the compact-throughput choice: it leads the compact indexes on all three local batch runs. IVF-RQ uses smaller files and raises recall from 0.8627 / 0.8577 / 0.8036 to 0.9148 / 0.9039 / 0.8203, but batch throughput falls by about 3.4–4.4×. IVF-PQ is smaller and faster than RQ, but its 0.5819–0.7410 recall makes it a capacity-first choice rather than a default accuracy compromise. DiskANN is compelling on SIFT and especially GIST: it reads below 1 MiB per query on both, and on GIST greatly outpaces IVF-FLAT. It is not automatically best on GloVe, where IVF-FLAT has higher recall, lower P95, and higher batch throughput at the recorded settings. Treat l_search and nprobe as calibration points whenever the displayed recall misses the production gate.

+
+
+ Historical remote and object-store models · July 2026 +

All rows in this archive, including IVF-SQ, retain the original implementation and 12-worker configuration. The current pooled-bound and cached IVF-SQ has not been remeasured under these latency models.

Remote cache with 2 ms per I/O round

@@ -200,6 +235,7 @@

Object store with 20 ms per I/O round

At 20 ms per round, IVF-RQ is the strongest measured compact one-round option: it reaches 0.90-class recall on SIFT/GIST and 0.8203 on GloVe. IVF-SQ is faster when its lower recall is enough, and IVF-PQ is smaller when stronger quantization loss is acceptable. IVF-FLAT now looks competitive on one-round SIFT/GloVe in this fixed-latency model, but that result assumes 28–33 MiB transfers have no bandwidth cost; GIST's 283 MiB and five rounds expose the boundary. DiskANN averages about one modeled round after warmup, but dependent graph rounds remain visible in P95. A complete local SSD cache remains its preferred deployment.

The automatic read plan affects approximate searchThe latency-derived local tier uses graph beam 4 while remote and object-store tiers use beam 16, so the same l_search can return different approximate candidates; this is visible for both GIST and GloVe at l_search=100. The tiers use 16 KiB, 32 KiB, and 64 KiB coalescing windows respectively. Storage latency itself does not change ground truth. Compare indexes with the same latency and capability hints when isolating media effects.
Remote-model boundaryThe vectors and exact neighbors are public corpus data, but the 2 ms and 20 ms profiles are controlled I/O models rather than measurements from a production cache or object store. They add fixed latency without modeling bandwidth, cache misses, TLS, retries, throttling, request limits, or tail-latency variance.
+
@@ -207,24 +243,25 @@

Object store with 20 ms per I/O round

Choose by constraint

There is no best index independent of data distribution. Narrow the field to one or two candidates, then evaluate Recall@K, P95/P99 latency, file size, build time, and object-store bytes on real queries.

-
Practical default orderFirst reject indexes that cannot meet the measured recall target. Build IVF-FLAT to establish the corpus-specific IVF ceiling. If a compact representation is required, choose IVF-SQ for throughput, IVF-RQ for recall, or IVF-PQ for minimum bytes. Evaluate DiskANN separately for immutable data served from local SSD; do not select it only because the collection is large or assume L2 results transfer to another metric.
+
Practical default orderFirst reject indexes that cannot meet the measured recall target. Build IVF-FLAT to establish the corpus-specific IVF ceiling. If a compact representation is required, start with IVF-SQ when its one-byte-per-dimension codes fit; compare IVF-RQ at the required bit budget and IVF-PQ for minimum bytes. Evaluate DiskANN separately for immutable data served from local SSD; do not select it only because the collection is large or assume L2 results transfer to another metric.

Measured recommendation matrix

-
Index / searchSIFT Recall / P95 / batch QPS / roundsGIST Recall / P95 / batch QPS / roundsGloVe Recall / P95 / batch QPS / rounds
IVF-PQ0.7142 / 6.16 ms / 7,282 / 1.00.7410 / 7.12 ms / 1,162 / 1.00.5819 / 5.94 ms / 8,243 / 1.0
+

Except for the refreshed SQ results, numerical evidence below comes from the July five-index matrix. The latest SQ results cover SIFT, GIST, and GloVe; other algorithms have not been rerun in this refresh.

+
Production constraintStart withEvidence from this runMove away when
- - + + - - + +
Production constraintStart withRecorded evidenceMove away when
Establish a recall ceiling or debug ranking qualityIVF-FLATHighest measured recall on all three corpora: 0.9937 / 0.9549 / 0.8832, with roughly four-second SIFT/GloVe builds.The 28–283 MiB selected-list reads or raw-vector file size exceed the serving budget.
Highest compact batch throughputIVF-SQ11,082 / 1,502 / 12,962 local batch QPS at 0.8627 / 0.8577 / 0.8036 recall; files are about one quarter of IVF-FLAT.The recall gate is above SQ, or one byte per dimension is still too large.
Strongest recall in a compact IVF fileIVF-RQ0.9148 / 0.9039 / 0.8203 recall in files smaller than IVF-SQ, with one sequential read round per query in all three modeled profiles.Batch throughput is the primary SLO; the four-bit scanner is 3–4.5× slower than SQ in the local run.
Compact scans and repeated-query throughputIVF-SQThe refreshed three-corpus SQ row records pooled-bound recall, native P95, batch QPS, and average payload I/O. Files are about one quarter of IVF-FLAT.The recall gate is above SQ, or one byte per dimension is still too large.
Configurable codes smaller than IVF-SQIVF-RQThe historical four-bit run reached 0.9148 / 0.9039 / 0.8203 recall in files smaller than IVF-SQ, with one sequential read round per query in all three modeled profiles.The chosen width misses the recall or throughput target. Compare the current SQ implementation before accepting the extra scan cost.
Minimum index file and compact-IVF scan bytesIVF-PQThe smallest files—0.032 / 0.230 / 0.030 GiB—and the smallest IVF selected-list reads at 1.84–17.84 MiB, with strong batch throughput.0.5819–0.7410 recall is below the gate; increase the PQ budget or choose SQ/RQ instead.
High-recall immutable data on local SSDDiskANN, checked against IVF-FLAT for the same metricThe recorded L2-equivalent run reached 0.9915 / 0.9336 / 0.8355 recall with 0.66 / 0.83 / 0.96 MiB reads; SIFT/GIST P95 is 1.50 / 1.83 ms.Metric-specific recall misses the gate, rebuilds are frequent, the file is not locally cached, preview maturity is unacceptable, or the corpus behaves like GloVe at l_search=100.
Frequent rebuilds or rapidly changing snapshotsIVF-FLAT, IVF-SQ, or IVF-RQThese build in about 4 seconds on SIFT/GloVe and 23–25 seconds on GIST; IVF-PQ is about 2× slower and DiskANN is 18–37× slower than IVF-FLAT.The serving phase dominates lifetime cost enough to justify PQ training or graph construction.
Direct 2/20 ms remote or object-store readsCompact IVF selected by recall: PQ → SQ → RQPQ and RQ use one sequential multi-range round here; SQ does so on SIFT/GloVe and averages 1.9 rounds on GIST. Choose successively more recall at greater bytes or CPU cost.Bandwidth, request limits, or real tail latency invalidate the fixed-latency model; prefer a complete local SSD cache and rerun the benchmark.
Frequent rebuilds or rapidly changing snapshotsIVF-FLAT, IVF-SQ, or IVF-RQThe historical five-index run favored these IVF variants for rebuild cost. Refreshed SQ builds take 0.886 s / 5.850 s / 0.797 s on SIFT / GIST / GloVe.The serving phase dominates lifetime cost enough to justify PQ training or graph construction.
Direct 2/20 ms remote or object-store readsCompact IVF selected by measured recall, payload bytes, and cache budgetIn the historical uncached run, PQ and RQ used one sequential multi-range round; SQ did so on SIFT/GloVe and averaged 1.9 rounds on GIST. Current SQ cache hits avoid payload reads; measure miss behavior on the real adapter.Bandwidth, request limits, or real tail latency invalidate the fixed-latency model; prefer a complete local SSD cache and rerun the benchmark.
Inner product or cosineIVF-FLAT as the recall control; DiskANN as an additional candidate for immutable local-SSD servingAll five implementations support L2, IP, and cosine. DiskANN normalizes cosine internally and uses metric-aware graph construction and exact reranking, but the displayed public-corpus matrix was recorded through the L2-equivalent benchmark path.The selected configuration misses its metric-specific recall gate—retune nprobe, representation width, OPQ, or l_search before deployment.
-
A displayed winner can still be the wrong choiceThese recommendations apply to the recorded nlist=1024, nprobe=64, PQ ratio, RQ bits, and l_search=100. For example, the current GloVe run does not reach 0.90 recall with any index, and current GIST reaches 0.95 only with IVF-FLAT. If a required recall threshold is not present in the table, tune and rebuild rather than choosing the closest result.
+
A displayed winner can still be the wrong choiceThese recommendations apply to the recorded nlist=1024, nprobe=64, PQ ratio, RQ bits, and l_search=100. For example, the historical GloVe run does not reach 0.90 recall with any index, and historical GIST reaches 0.95 only with IVF-FLAT. The new SQ GloVe result also remains below 0.90. If a required recall threshold is not present in the table, tune and rebuild rather than choosing the closest result.

I need a trustworthy baseline

Start with IVF-FLAT. It exposes the IVF partition ceiling without quantization loss and rebuilds quickly.

Explore IVF-FLAT →
-

I need compact high recall

Choose IVF-RQ when its 0.82–0.91 measured recall matters more than batch throughput; compare every result with the IVF-FLAT ceiling.

Explore IVF-RQ →
+

I need configurable compact codes

Try IVF-RQ when SQ is too large and tune the bit width against recall and throughput. Compare every result with the IVF-FLAT ceiling and the current SQ baseline.

Explore IVF-RQ →

I need the smallest index

Choose IVF-PQ when its corpus-specific recall passes the gate. It is the capacity-first option, not the automatic middle ground.

Explore IVF-PQ →
-

I need compact batch speed

Choose IVF-SQ when one byte per dimension fits and its measured 0.80–0.86 recall is enough; it is the fastest compact scanner here.

Explore IVF-SQ →
+

I need compact batch speed

Choose IVF-SQ when one byte per dimension fits. Pooled training bounds and bounded partition caching improve the measured three-corpus recall and repeated-query performance; validate the target corpus and cache budget.

Explore IVF-SQ →

Raw vectors exceed RAM but fit local SSD

Evaluate DiskANN for immutable L2, IP, or cosine data when high recall and sub-MiB query reads justify a much slower build; retain IVF-FLAT as the metric-specific accuracy control.

Explore DiskANN →

Data lives in S3, OSS, or HDFS

Prefer durable publication plus a complete local SSD cache. For direct remote reads, start with a compact IVF index when one-round scans meet recall; use DiskANN only after measuring its corpus-dependent coalesced graph rounds.

Compare deployment modes →
@@ -250,7 +287,7 @@

Measured recommendation matrix

deployment-profileBuildDiskANNSelects interleaved layout for eligible memory/local serving and compact layout for remote/object servingExplicit layout/encoding/build-distance overrides always win estimated_random_read_latency_nanosReader input capabilityDiskANNSelects the internal read window, graph beam, and automatic cache partition without probe I/O0 measures the mandatory header read; positive values are useful for known remote/cache latency l_searchQueryDiskANNLarger DiskANN candidate list, usually higher recall and latencyAuto uses calibrated 100/200/400 when available, otherwise max(100, 2k) - memory_budget_bytesReaderDiskANNControls required resident state plus automatically partitioned adjacency/raw-vector caches4 GiB; cache sub-budgets are internal + memory_budget_bytesReaderDiskANN, IVF-SQReserves resident state, then bounds DiskANN adjacency/raw-vector caches or the IVF-SQ partition cache4 GiB; cache sub-budgets are internal
diff --git a/docs/ivf-flat.html b/docs/ivf-flat.html index aad9289..5dc76b1 100644 --- a/docs/ivf-flat.html +++ b/docs/ivf-flat.html @@ -92,7 +92,7 @@

Capacity estimate

Tuning order

-
  1. Fix the metric. Build exact ground truth with the production metric and include zero-vector edge cases for cosine.
  2. Start automatic. Supply the final corpus count, inspect the resolved nlist, and use automatic query width.
  3. Calibrate only if needed. Sweep explicit nprobe around the automatic value and record Recall@K, P95/P99, selected lists, and bytes read.
  4. Measure batch search. Readers submit bounded multi-range batches, which often matters more than single-query latency on object stores.
  5. Only then compress. Try IVF-SQ for a one-byte-per-dimension scan, IVF-PQ for stronger compression, or IVF-RQ for the strongest measured compact-IVF recall.
+
  1. Fix the metric. Build exact ground truth with the production metric and include zero-vector edge cases for cosine.
  2. Start automatic. Supply the final corpus count, inspect the resolved nlist, and use automatic query width.
  3. Calibrate only if needed. Sweep explicit nprobe around the automatic value and record Recall@K, P95/P99, selected lists, and bytes read.
  4. Measure batch search. Readers submit bounded multi-range batches, which often matters more than single-query latency on object stores.
  5. Only then compress. Try IVF-SQ for a one-byte-per-dimension scan, IVF-PQ for stronger compression, or IVF-RQ for configurable multi-bit compression. Compare current SQ and RQ recall at the required storage budget.
diff --git a/docs/ivf-rq.html b/docs/ivf-rq.html index 76424f8..add8457 100644 --- a/docs/ivf-rq.html +++ b/docs/ivf-rq.html @@ -18,7 +18,7 @@

Positioning and trade-offs

Default codepadded_d / 2 bytes
Per-vector factors5 × f32
Learned modelIVF centroids only
DimensionAny positive value
-

Good fit

  • You need higher recall than IVF-SQ at a smaller serialized size.
  • Training time and model complexity should stay close to IVF-FLAT.
  • The source supports one concurrent multi-range read for selected lists.
  • Approximate in-list ranking is acceptable and measured against real ground truth.

Poor fit

  • Raw-vector or exact reranking accuracy is mandatory.
  • Sub-millisecond local latency matters more than 0.90-class recall.
  • The collection is highly mutable; this implementation writes immutable files.
  • Resident PQ tables and lower recall are acceptable in exchange for a still smaller IVF-PQ index.
+

Good fit

  • You need smaller codes than IVF-SQ while meeting the measured recall target.
  • Training time and model complexity should stay close to IVF-FLAT.
  • The source supports one concurrent multi-range read for selected lists.
  • Approximate in-list ranking is acceptable and measured against real ground truth.

Poor fit

  • Raw-vector or exact reranking accuracy is mandatory.
  • Sub-millisecond local latency matters more than 0.90-class recall.
  • The collection is highly mutable; this implementation writes immutable files.
  • Resident PQ tables and lower recall are acceptable in exchange for a still smaller IVF-PQ index.
Why it remains IVF-RQThe public index family name is unchanged. The pre-release 1-bit/query-bit experiment was replaced completely: bit width is now a property of persisted data, not a per-query switch.
@@ -94,7 +94,7 @@

Capacity estimate

Tuning order

-
  1. Run IVF-FLAT with the target nlist/nprobe to establish the partition recall ceiling.
  2. Start IVF-RQ at the default four bits.
  3. If recall is low for both indexes, increase nprobe. If only IVF-RQ is low, try five bits before increasing I/O through more lists.
  4. Compare IVF-SQ when simpler/faster scans matter; compare IVF-PQ when minimum size matters.
  5. Validate the final choice on a public or production corpus, including batch and the real storage adapter.
+
  1. Run IVF-FLAT with the target nlist/nprobe to establish the partition recall ceiling.
  2. Start IVF-RQ at the default four bits.
  3. If recall is low for both indexes, increase nprobe. If only IVF-RQ is low, try five bits before increasing I/O through more lists.
  4. Compare the current IVF-SQ recall and scan results when one byte per dimension fits; compare IVF-PQ when minimum size matters.
  5. Validate the final choice on a public or production corpus, including batch and the real storage adapter.
diff --git a/docs/ivf-sq-performance.md b/docs/ivf-sq-performance.md new file mode 100644 index 0000000..3ab3671 --- /dev/null +++ b/docs/ivf-sq-performance.md @@ -0,0 +1,208 @@ + + +# IVF-SQ versus Lance: September 2026 + +SIFT1M and GloVe improve in build time, Python query latency, batch throughput, +and recall relative to Lance 11.0.0. The GIST1M rerun improves build time and +recall, with a modest 5% lower median Python P95, but **batch throughput remains +26% below Lance**. The goal of beating Lance in every measured workload is not +yet met. These are local measurements with the configuration below. + +The site pages summarize [current results](ivf-sq.html#benchmarks), +[reader options](api.html#reader-options), and [benchmark setup](development.html#ivfsq-lance). + +## Results + +Medians of three runs, Apple M4 Pro (12 CPU cores, 48 GiB RAM), eight workers, +release Rust 1.95.0, 1,024 partitions, 64 probes, Top-10, 65,536 training rows, +and the first 1,000 independent ANN-Benchmarks queries. GloVe base/query +vectors were normalized identically before either engine read them. + +| Corpus | Engine | Index build | Python query P95 | Python batch QPS | Recall@10 | +| --- | --- | ---: | ---: | ---: | ---: | +| SIFT1M, 1M × 128 | Paimon IVF-SQ | 0.886 s | 0.260 ms | 9,927 | 0.9812 | +| SIFT1M | Lance IVF-SQ 11.0.0 | 3.613 s | 1.099 ms | 4,181 | 0.9772–0.9775 | +| GIST1M, 1M × 960 | Paimon IVF-SQ | 5.850 s | 1.769 ms | 986 | 0.9399 | +| GIST1M | Lance IVF-SQ 11.0.0 | 18.651 s | 1.862 ms | 1,334 | 0.9249 | +| GloVe, 1,183,514 × 100 | Paimon IVF-SQ | 0.797 s | 0.240 ms | 11,052 | 0.8760 | +| GloVe | Lance IVF-SQ 11.0.0 | 2.790 s | 1.089 ms | 4,240 | 0.7843–0.7845 | + +In SIFT / GIST / GloVe order, construction is **4.08× / 3.19× / 3.50× as fast** +as Lance, Python P95 is **76% / 5% / 78% lower**, and Python batch throughput +is **2.37× / 0.74× / 2.61×** Lance's. GIST therefore retains a batch-throughput +gap. Its three P95 observations span 1.692–1.857 ms for Paimon and +1.823–2.045 ms for Lance with parallelism 8; the ranges overlap, so the small +median latency advantage should not be treated as a large or universal win. +Lance recall ranges show the medians of the two scheduling modes. Its faster +setting is used separately for each performance metric: partition parallelism +8 for single-query latency and the default 0 for batch throughput. No raw-vector +refinement is requested in either engine. Returned data consists of IDs and +distances; neither benchmark requests the raw vector column. + +The [recorded measurements](benchmarks/ivfsq-lance-20260906.json) include all +three repetitions and both Lance scheduling modes. Both engines use their native training algorithms and iteration policies; +training sample counts match, but learned centroids need not. Lance training is not +bitwise deterministic across builds, so recall varies slightly between runs. +Paimon's Rust and Python result paths can choose different members of a tied +boundary; SIFT and GIST recall differ by 0.0001 between native and Python +single-query entry points. GIST Python batch recall is 0.9400. The GIST rerun +uses three interleaved baseline/current/Python/Lance repetitions on the same day. + +## Changes and baseline + +The baseline is commit `8dcabf208c99707eab3938af0bcc40530fdb4cad`. + +| Rust ann_bench metric | SIFT baseline → current | GIST baseline → current | GloVe baseline → current | +| --- | ---: | ---: | ---: | +| Build | 934 → 886 ms | 6,404 → 5,850 ms | 849 → 797 ms | +| Encode/add | 549 → 538 ms | 3,705 → 3,613 ms | 539 → 517 ms | +| Serialize | 125 → 93 ms | 935 → 563 ms | 118 → 90 ms | +| Query P95 | 840 → 298 µs | 4,386 → 1,828 µs | 739 → 282 µs | +| Batch QPS | 6,417 → 8,987 | 899 → 998 | 6,988 → 10,009 | +| Recall@10 | 0.8626 → 0.9811 | 0.8576 → 0.9400 | 0.8036 → 0.8760 | +| File bytes | 131,359,859 → unchanged | 973,626,471 → unchanged | 121,822,316 → unchanged | + +GIST serialization time falls by 40%, native query P95 is 58% lower, and native +batch throughput is 11% higher than the same eight-worker baseline. Peak process +RSS falls from 5.08 to 4.87 GiB. These improvements do not close the batch gap to +Lance shown above. + +1. **Avoid sparse-partition clipping.** Training previously estimated a separate + minimum and maximum from each partition's often tiny sample. Constant sample + dimensions and narrow extrema clipped unseen residuals. The trainer now pools + per-dimension residual extrema across partitions. Holding centroids fixed, + this alone raised SIFT Recall@10 from about 0.863 to 0.981. Broad pooled bounds + can reduce resolution on corpora with extreme outliers; measure such data + before adopting this training policy. +2. **Fuse encoding and bound training.** Partition-local reductions avoid the + training residual matrix. Encoding computes inverse scales once per partition, + subtracts the centroid in registers, and packs rounded NEON/AVX2 results + directly into the destination. Serialization transposes partitions in parallel + within 16 MiB batches (one oversized partition is processed alone). +3. **Reuse query heaps and prune safely.** Batch scans retain one heap per query. + An L2 block can stop after its first half only if every nonnegative partial + distance already exceeds or equals the current cutoff. Competitive distances + are still evaluated completely. A first partition supplies the cutoff for + parallel single-query scans. A one-query batch uses the single-query path. +4. **Reuse decoded partitions within the existing memory budget.** The unified + reader and bindings use a bounded FIFO cache. Cache hits share immutable + buffers with `Arc`, bypass positional I/O, and avoid decoding IDs again. + Metadata, cache slot/queue storage, and retained payload capacities are + reserved before retaining entries. Filters and distances are never cached. + Oversized streamed partitions bypass the cache. Zero budget disables caching; + required metadata still loads. Direct `IVFSQIndexReader::open` stays uncached; + `open_with_options` enables the cache. + +The SIFT/GloVe uncached optimized intermediate also improved Rust batch +throughput to 9,176 / 10,224 QPS at the new higher recall; this intermediate +was not rerun on GIST. Cache-enabled ann_bench batch +numbers are slightly lower because it opens a fresh reader and times a complete +first batch, including payload reads and cache insertion. The Python comparison +warms the selected partitions before timing both engines. Its native file adapter +uses ordinary `os.pread` callbacks; no special in-memory adapter is used. + +Paimon's configured reader budget is 4 GiB and Lance's index cache is 1 GiB +(explicit `index_cache_size_bytes`, not the deprecated entry-count parameter); +each entire measured index fits within either budget. Neither reserves that +whole amount as payload memory. Paimon's peak process RSS was about +787 / 4,986 / 733 MiB for SIFT / GIST / GloVe, including the benchmark's source data and build phases. Build time includes +training, assignment, encoding and index serialization. Lance additionally needs +a source dataset; its data-writing time is recorded separately and **excluded** +from the comparison above. Paimon build stages use the Rust benchmark; query +latency/throughput in the first table use both public Python interfaces. + +The IVSQ v1 format, flags, row-ID encoding, and golden fixture bytes remain +compatible. Old files benefit from the reader changes immediately. Rebuilding +is required to get the new training bounds; existing files keep their recorded +per-partition quantizers. + +## Reproduce + +Obtain the public SIFT, GIST, and GloVe HDF5 files from +[ANN-Benchmarks](https://github.com/erikbern/ann-benchmarks). Install `numpy`, +`h5py`, `pyarrow`, and `pylance==11.0.0` in a temporary environment. Run each +benchmark separately, without concurrent compilation or other benchmarks. Run from +the repository root, and set `DATA` to the directory containing the downloaded +HDF5 files and `OUT` to the directory for generated indexes before running the +commands below. + +```sh +python tools/convert_ann_benchmarks.py "$DATA/sift-128-euclidean.hdf5" "$DATA/sift" \ + --prefix sift --query-limit 1000 +python tools/convert_ann_benchmarks.py "$DATA/gist-960-euclidean.hdf5" "$DATA/gist" \ + --prefix gist --query-limit 1000 +python tools/convert_ann_benchmarks.py "$DATA/glove-100-angular.hdf5" "$DATA/glove" \ + --prefix glove --query-limit 1000 --normalize-l2 +cargo bench -p paimon-vindex-core --bench ann_bench --no-run +cargo build --release -p paimon-vindex-ffi + +# Repeat for corpus=gist and corpus=glove, and repeat each engine three times. +corpus=sift +export RAYON_NUM_THREADS=8 ANN_INDEXES=IVF_SQ ANN_TRAIN_N=65536 +export ANN_NLIST=1024 ANN_NPROBE=64 ANN_K=10 ANN_STORAGE_CASES=local_ssd_warm_cache +export ANN_BASE_FVECS="$DATA/$corpus/${corpus}_base.fvecs" +export ANN_QUERY_FVECS="$DATA/$corpus/${corpus}_query.fvecs" +export ANN_GROUND_TRUTH_IVECS="$DATA/$corpus/${corpus}_ground_truth.ivecs" +export ANN_KEEP_INDEXES=1 ANN_OUTPUT_DIR="$OUT/paimon-$corpus" +cargo bench -p paimon-vindex-core --bench ann_bench + +python tools/benchmark_lance_ivfsq.py \ + --base "$ANN_BASE_FVECS" --queries "$ANN_QUERY_FVECS" \ + --ground-truth "$ANN_GROUND_TRUTH_IVECS" --output-dir "$OUT/lance-$corpus" \ + --threads 8 --train-n 65536 --nlist 1024 --nprobe 64 --nq 1000 --k 10 \ + --query-parallelism 0 8 --cache-bytes 1073741824 --repeats 3 + +# INDEX is the ivf_sq.index preserved under ANN_OUTPUT_DIR//. +PYTHONPATH=python PAIMON_VINDEX_LIB_PATH="$PWD/target/release/libpaimon_vindex_ffi.dylib" \ +python tools/benchmark_ivfsq_reader.py --index "$INDEX" \ + --queries "$ANN_QUERY_FVECS" --ground-truth "$ANN_GROUND_TRUTH_IVECS" \ + --threads 8 --nprobe 64 --nq 1000 --k 10 \ + --memory-budget-bytes 4294967296 --repeats 3 +``` + +Use `--memory-budget-bytes 0` on the Python reader benchmark to isolate the +uncached scan path; this does not flush the operating-system cache. For IVF-SQ, +`optimize_for_search` and `warmup_queries` initialize metadata only. Replay actual +searches to warm the partition cache, as the Python script does. On Linux, use the `.so` native library instead of `.dylib`. +The Lance script creates a new dataset per repetition and reports the path; +remove those generated benchmark outputs when no longer needed. + +## Verification and limits + +- Workspace tests: 510 passed, 2 intentionally ignored; includes v1 golden fixtures. +- Python bindings: 28 passed. +- The pre-change reader at `8dcabf2` opened newly generated SIFT, GIST, and + GloVe indexes and completed 1,000 single queries plus batch search per corpus. + Recall@10 differed from the new reader by at most 0.0002; this verifies file + compatibility, not identical ordering of every result. +- x86_64 build and SQ tests under Rosetta: 33 passed. Rosetta reported AVX2/FMA + unavailable, so AVX2 was compiled but its runtime kernel needs native x86 CI. +- `cargo fmt`, workspace Clippy with warnings denied, license headers, and diff + whitespace checks passed. +- Added regression coverage for packed-encoding rounding/tails, residual extrema, + sparse/empty partition calibration, L2 cutoff boundaries, single-query and + seeded batch paths, cache reuse, eviction, budget bypass, filtering, and read + failures followed by retries. + +This run does not establish superiority on Linux/x86, cold storage, +object-store latency, or every metric/distribution. GIST batch throughput remains +below Lance. The homepage build/local tables contain the refreshed IVF-SQ rows; +other indexes retain their labeled July measurements, and the historical remote +models and implementation notes remain in collapsible archives. diff --git a/docs/ivf-sq.html b/docs/ivf-sq.html index 3cf58e0..ddab505 100644 --- a/docs/ivf-sq.html +++ b/docs/ivf-sq.html @@ -8,12 +8,13 @@ IVF-SQ · Paimon Vector Index +
VIPaimon Vector Index
-
-

Per-list residual scalar quantization

IVF-SQ

Store one unsigned byte per residual dimension and scan the selected IVF lists directly. IVF-SQ targets the gap between raw IVF-FLAT and aggressively compressed IVF-PQ/RQ without paying for a graph in every list.

1 byte / dimensionPer-list boundsSIMD scanMagic: IVSQ
-
+
+

Residual scalar quantization

IVF-SQ

Store one unsigned byte per residual dimension and scan the selected IVF lists directly. IVF-SQ targets the gap between raw IVF-FLAT and aggressively compressed IVF-PQ/RQ without paying for a graph in every list.

1 byte / dimensionPooled training boundsSIMD scanMagic: IVSQ
+
-
+

Position

IVF-SQ preserves the IVF partitioning model and replaces every residual f32 component with an 8-bit scalar code. It usually occupies about one quarter of IVF-FLAT's vector payload. Unlike IVF-PQ, each dimension is quantized independently, so there is no subquantizer-count parameter or codebook lookup table.

@@ -21,7 +22,7 @@

Position

Build and search

-
  1. Train the IVF coarse centroids and assign training vectors to lists.
  2. Subtract each list centroid and learn per-dimension minimum/maximum residual bounds for that list.
  3. Encode every residual coordinate to an unsigned byte. Empty training lists use the global residual bounds.
  4. At query time, select nprobe lists, load their sorted row IDs and codes in one multi-range read, scan the codes with SIMD L2 or inner-product kernels, and merge the top K.
+
  1. Train the IVF coarse centroids and assign training vectors to lists.
  2. Compute per-dimension residual extrema using partition-local reductions, then pool them across the training sample. This avoids clipping unseen vectors to the narrow or constant bounds of sparsely sampled partitions.
  3. Encode every residual coordinate to an unsigned byte. New indexes use pooled residual bounds for every list; existing v1 files retain their recorded per-list bounds.
  4. At query time, select nprobe lists, reuse cached partitions, and load missing sorted row IDs and codes in bounded multi-range batches. Scan the codes with the metric-specific kernel and retain the top K. Blocked L2 scans use SIMD and a conservative partial-distance cutoff.

Cosine input is normalized through the shared metric preprocessing path. Filters are checked while scanning, so excluded rows do not enter the top-K heap.

@@ -48,29 +49,45 @@

Parameters

Stable v1 storage

64 B headerIVSQ v1
Global bounds2 × d × f32
Per-list bounds2 × nlist × d × f32
IVF centersnlist × d × f32
Offset tablenlist × 16 B
Listsblocked SQ codes + delta IDs

Every non-empty list is sorted by signed row ID before writing. Codes come first and are transposed within up-to-32-row blocks, with dimension before row lane, so SIMD evaluates multiple candidates together and the reader scans directly from the list payload allocation. The trailing IDs use the shared delta-varint encoding and remain aligned with code lanes. The normative byte layout and golden fixture are in the storage-format specification.

+
Upgrading existing indexesThe header, flags, code layout, and row-ID encoding remain IVSQ v1. Existing files gain the reader optimizations without rebuilding and keep their recorded quantization bounds. Retrain and rebuild to obtain pooled bounds and their measured recall improvement. New files store the pooled bounds in the existing per-list metadata fields.

Open-source comparison

-

Faiss IVF-SQ provides residual SQ4/SQ6/SQ8/F16 encodings, parallel add, and query-parallel scanning over generic inverted lists. Milvus Knowhere builds on the same scanner model and adds concurrent inverted-list mutation. This implementation deliberately fixes the first immutable format at SQ8, uses per-list per-dimension residual bounds, stores compressed sorted IDs instead of fixed eight-byte IDs, and transposes each 32-row code block for its CPU SIMD kernels.

-

The comparison did produce two build changes: non-cosine inputs are borrowed instead of copied, and assigned lists are encoded in parallel with one reusable residual vector per worker. It did not justify changing the persisted layout. SQ4/SQ6 would overlap IVF-PQ/RQ, quantile clipping would introduce another corpus-sensitive accuracy parameter, and compressing the relatively small resident bounds would save little beside the N × d code payload. The existing codes-first payload already allows one bounded multi-range operation and reuse of the read allocation as the scan buffer.

+

Faiss IVF-SQ provides residual SQ4/SQ6/SQ8/F16 encodings, parallel add, and query-parallel scanning over generic inverted lists. Milvus Knowhere builds on the same scanner model and adds concurrent inverted-list mutation. This implementation deliberately fixes the first immutable format at SQ8, stores per-dimension residual bounds in each list’s metadata, stores compressed sorted IDs instead of fixed eight-byte IDs, and transposes each 32-row code block for its CPU SIMD kernels.

+

Non-cosine inputs are borrowed, residual extrema are reduced in parallel without a training residual matrix, and assigned rows are encoded directly with precomputed scales and packed NEON/AVX2 conversion. The persisted layout is unchanged. SQ4/SQ6 would overlap IVF-PQ/RQ, quantile clipping would introduce another corpus-sensitive accuracy parameter, and compressing the relatively small resident bounds would save little beside the N × d code payload. The existing codes-first payload already allows one bounded multi-range operation and reuse of the read allocation as the scan buffer.

I/O and batching

-

Open reads the fixed header and contiguous resident metadata in two positional operations; the outer type dispatcher adds one small magic read. A query submits selected list ranges through the abstract positional-read interface in capability- and 64 MiB-bounded multi-range batches. SIFT1M and GloVe-100 use one payload round per query at nprobe=64; 960-dimensional GIST1M averages 1.9. Batch search first deduplicates the lists selected across queries, loads each unique list once across the bounded rounds, and then scans queries in parallel. This is especially useful when a remote adapter executes the supplied ranges concurrently.

-

The add path retains the caller's L2/IP slice without copying it, partitions assigned row positions by list, and encodes those lists in parallel. Each active list task reuses one d-component residual buffer and one code buffer; the add path never materializes an N × d residual matrix. The writer retains only each list's row-order permutation and encoded IDs. It generates one list's blocked codes directly from the in-memory row-major codes, writes them, and releases the temporary buffer before processing the next list.

-

During scanning, a candidate whose distance cannot improve a full Top-K heap is rejected before row-ID hashing. Batch remains query-parallel after a measured list-major experiment regressed SIFT/GIST throughput; list payloads are still deduplicated and read once.

-

IVF-SQ still reads complete selected lists. At high nprobe, scan bytes grow linearly; DiskANN is the better fit when the workload requires small page-granular reads from a large local-SSD index.

+

Open reads the fixed header and contiguous resident metadata in two positional operations; the unified type dispatcher reuses that header. On a cache miss, a query submits selected list ranges through the abstract positional-read interface in capability- and 64 MiB-bounded multi-range batches. The historical uncached measurements below use one payload round per query for SIFT1M and GloVe-100 at nprobe=64; GIST1M averages 1.9. Batch search first deduplicates the lists selected across queries, loads each missing unique list once across the bounded rounds, and then scans queries in parallel. This is especially useful when a remote adapter executes the supplied ranges concurrently.

+

The add path retains the caller's L2/IP slice without copying it, partitions assigned row positions by list, and encodes those lists in parallel. Each active list task precomputes one d-component scale vector and fuses residual subtraction with quantization directly into the destination codes; the add path never materializes an N × d residual matrix. The writer retains only each list's row-order permutation and encoded IDs. It transposes lists in parallel within 16 MiB batches, writes them in physical order, then releases those buffers. A list larger than the batch budget is processed on its own.

+

During scanning, a candidate whose distance cannot improve a full Top-K heap is rejected before row-ID hashing. Batch scanning keeps one heap per query across loaded lists, avoiding per-list heaps and result merging. L2 scans may reject a complete 32-row block when its nonnegative partial distances cannot improve the current heap; returned candidates retain their complete distances. Single-query batches use the ordinary partition-parallel search path.

+

VectorIndexReader::open, open_with_options, and the language bindings use the reader memory budget (4 GiB by default) for a FIFO cache of decoded partitions. Resident metadata, cache slots, queue storage, and retained payload capacities are charged before insertion. Hits share immutable payloads without copying or positional I/O; filters and scores remain query-local. Oversized streamed lists bypass the cache. Zero budget disables caching, while required metadata still loads; transient query allocations are outside this retained-cache limit. Direct IVFSQIndexReader::open retains uncached behavior; use its open_with_options to enable caching. See reader options.

+

Populate the cache by replaying representative queries through search or search_batch. For IVF-SQ, optimize_for_search and warmup_queries only ensure resident metadata is loaded; they do not prefetch partitions.

+

On cache misses, IVF-SQ still reads complete selected lists. At high nprobe, scan bytes grow linearly; DiskANN is the better fit when the workload requires small page-granular reads from a large local-SSD index.

Public benchmarks

-

On the documented Apple M4 Pro run with one million-scale public vectors, nlist=1024, nprobe=64, k=10, and 12 Rayon workers:

+

September 2026: IVF-SQ versus Lance 11.0.0

+

Three-run medians on Apple M4 Pro (12 CPU cores, 48 GiB RAM), with eight workers, nlist=1024, nprobe=64, k=10, 65,536 training rows, and 1,000 held-out public queries. Both engines read the same vectors; GloVe is normalized before either engine loads it. Query measurements use the public Python interfaces with selected partitions warmed and no raw-vector refinement.

+
+ + + + + + +
CorpusEngineIndex buildPython P95Python batch QPSRecall@10
SIFT1MPaimon IVF-SQ0.886 s0.260 ms9,9270.9812
SIFT1MLance 11.0.03.613 s1.099 ms4,1810.9772–0.9775
GIST1MPaimon IVF-SQ5.850 s1.769 ms9860.9399
GIST1MLance 11.0.018.651 s1.862 ms1,3340.9249
GloVe-100Paimon IVF-SQ0.797 s0.240 ms11,0520.8760
GloVe-100Lance 11.0.02.790 s1.089 ms4,2400.7843–0.7845
+

In SIFT / GIST / GloVe order, Paimon builds 4.08× / 3.19× / 3.50× as fast, lowers median Python P95 by 76% / 5% / 78%, and delivers 2.37× / 0.74× / 2.61× Lance's batch throughput. Build time includes training, encoding, and serialization; Lance source-dataset writing is excluded. Lance uses its faster scheduling mode for each metric: partition parallelism 8 for P95 and 0 for batch QPS; recall ranges cover both modes.

+

Paimon's reader budget is 4 GiB and Lance's index-cache budget is 1 GiB; each measured index fits both. Both engines use native training defaults with the same sample count. GIST batch throughput remains about 26% below Lance, and its three-run P95 ranges overlap despite the 5% lower median. These results do not establish performance on cold storage, object stores, or other architectures and distributions. Pooled bounds can lose resolution on extreme-outlier data. See the full report and verification, raw measurements, and reproduction guide.

+

Historical uncached implementation

+

The following measurements predate pooled bounds and partition caching. They use nlist=1024, nprobe=64, k=10, and 12 Rayon workers on Apple M4 Pro. Keep them separate from the current eight-worker comparison above, which now includes a GIST rerun.

DatasetRecall@10Build / peak RSSWarm P95 / batch QPSRead/query
SIFT1M0.86273.93 s / 0.79 GiB0.79 ms / 11,0828.38 MiB
GIST1M0.857722.7 s / 5.09 GiB3.56 ms / 1,50270.95 MiB
GloVe-1000.80363.86 s / 0.71 GiB0.71 ms / 12,9626.99 MiB

Compared with the immediately preceding implementation on the same files, peak RSS dropped by 56–61%, local P95 improved by 3–8%, and batch throughput improved by about 8–61%, depending on dimension and cache behavior. File bytes and read bytes are unchanged.

-
+
Apache Paimon Vector IndexIVF-SQ · v1
diff --git a/docs/releases.html b/docs/releases.html index a45d83e..531e309 100644 --- a/docs/releases.html +++ b/docs/releases.html @@ -49,6 +49,8 @@

Distribution channels

Upcoming: 0.5.0

The repository is currently developing the 0.5.0 line. Until an ASF vote passes and the signed source archive appears under Apache downloads, code and packages from this line are development artifacts rather than an Apache release.

+

IVF-SQ now pools residual training bounds, fuses residual encoding, transposes output in bounded parallel batches, and reuses query heaps with conservative L2 block pruning. Unified readers and language bindings also cache decoded partitions within the existing reader memory budget. The SIFT1M/GIST1M/GloVe comparison with Lance 11.0.0 records build, Python query, and recall results with reproducible commands.

+

IVSQ v1 files remain compatible. Existing indexes receive reader optimizations without a rebuild; retrain and rebuild to use the new quantization bounds. Set the reader memory budget to zero to disable the SQ cache; direct Rust IVFSQIndexReader::open remains uncached. See reader options and upgrade details.

Rust IVF API migrationVersion 0.5.0 makes quantizer_centroids private on IVFFlatIndex, IVFPQIndex, IVFSQIndex, and IVFRQIndex. Replace direct reads with quantizer_centroids() and direct assignments with set_quantizer_centroids(...). The setter validates the centroid shape, rejects replacement after vectors are added, and refreshes cached derived state. IVF variants of VectorIndexConfig also require use_approximate_coarse_assignment; set it to true for the automatic 0.5.0 behavior or false for exact nearest-centroid assignment. Option-map callers can select the same policy with ivf.coarse-assignment=auto|exact. The policy is fixed when the writer is created; direct IVF indexes do not expose a post-training policy switch. These are source-level changes; the stored index format is unchanged.
diff --git a/tools/README.md b/tools/README.md index 17cac1c..78f85ac 100644 --- a/tools/README.md +++ b/tools/README.md @@ -53,6 +53,19 @@ neighbor ordering as cosine distance. Published neighbor IDs are copied unchanged. Conversion fails if a vector is zero-length or has a non-finite norm. +## Lance IVF-SQ comparison + +`benchmark_lance_ivfsq.py` benchmarks Lance 11+ against the same converted +`fvecs`/`ivecs` as `ann_bench`. It requires `pylance`, `numpy`, and `pyarrow`. +It creates a fresh dataset/index per repetition, warms selected partitions, +tests both default and partition-parallel query scheduling, and emits JSONL +with build time, file size, recall, P50/P95, and sequential/batch throughput. +`benchmark_ivfsq_reader.py` measures the matching public Paimon Python reader +and accepts a zero memory budget to isolate uncached performance. +Raw dataset writing is reported separately from index construction. See +[the IVF-SQ performance report](../docs/ivf-sq-performance.md) for commands, +measured results, and the differences between the public API entry points. + ## Java staging deploy `deploy_java_staging.sh` deploys the Java release candidate artifacts to Apache diff --git a/tools/benchmark_ivfsq_reader.py b/tools/benchmark_ivfsq_reader.py new file mode 100644 index 0000000..226f23b --- /dev/null +++ b/tools/benchmark_ivfsq_reader.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Measure the public Python IVF-SQ reader, including positional-I/O callbacks. + +Build the native library in release mode, set PAIMON_VINDEX_LIB_PATH, and put +this checkout's python directory on PYTHONPATH. Uses the same fvecs/ivecs as +ann_bench and benchmark_lance_ivfsq.py. Emit one JSON row per repetition. +""" + +import argparse +import json +import os +from pathlib import Path +import time + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--index", type=Path, required=True) + parser.add_argument("--queries", type=Path, required=True) + parser.add_argument("--ground-truth", type=Path, required=True) + parser.add_argument("--nprobe", type=int, default=64) + parser.add_argument("--nq", type=int, default=1000) + parser.add_argument("--k", type=int, default=10) + parser.add_argument("--threads", type=int, default=8) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--memory-budget-bytes", type=int, default=4 * 1024**3) + args = parser.parse_args() + if min(args.nprobe, args.nq, args.k, args.threads, args.repeats) <= 0 or args.memory_budget_bytes < 0: + parser.error("counts must be positive; memory budget must be nonnegative") + os.environ["RAYON_NUM_THREADS"] = str(args.threads) + import numpy as np + from paimon_vindex import SearchParams, VectorIndexReader + + def read_vectors(path, dtype): + raw = np.memmap(path, mode="r", dtype=" args.nlist for p in args.nprobe): + parser.error("nprobe must be in 1..nlist") + if args.cache_bytes < 0 or any(p < 0 for p in args.query_parallelism): + parser.error("cache size and query parallelism must be nonnegative") + # Set pools before importing either native runtime. + for key in ["LANCE_CPU_THREADS", "LANCE_IO_THREADS", "RAYON_NUM_THREADS"]: + os.environ[key] = str(args.threads) + os.environ["OPENBLAS_NUM_THREADS"] = "1" + import lance + import numpy as np + import pyarrow as pa + + def read_vectors(path, dtype): + raw = np.memmap(path, mode="r", dtype=" len(base): + parser.error("inconsistent dimensions or train-n exceeds the base") + if not np.isfinite(base).all() or not np.isfinite(queries).all(): + parser.error("vectors must be finite") + if np.any(truth < 0) or np.any(truth >= len(base)): + parser.error("ground-truth IDs must refer to base rows") + table = pa.table({"vector": pa.FixedSizeListArray.from_arrays( + pa.array(base.ravel()), base.shape[1])}) + args.output_dir.mkdir(parents=True, exist_ok=True) + + def recall(results): + return sum(len(set(r) & set(g)) for r, g in zip(results, truth)) / truth.size + + for repeat in range(args.repeats): + path = Path(tempfile.mkdtemp(prefix="lance-ivfsq-", dir=args.output_dir)) / "data.lance" + started = time.perf_counter() + dataset = lance.write_dataset(table, str(path)) + data_write_s = time.perf_counter() - started + started = time.perf_counter() + dataset.create_index("vector", "IVF_SQ", metric="L2", + num_partitions=args.nlist, + sample_rate=args.train_n // args.nlist) + index_build_s = time.perf_counter() - started + dataset = lance.dataset(str(path), index_cache_size_bytes=args.cache_bytes) + index_bytes = sum(p.stat().st_size for p in (path / "_indices").rglob("*") if p.is_file()) + for nprobe in args.nprobe: + for parallelism in args.query_parallelism: + def search(q): + return dataset.to_table(columns=["_distance"], with_row_id=True, + nearest={"column": "vector", "q": q, + "k": args.k, "nprobes": nprobe, + "query_parallelism": parallelism}) + # Warm the union of selected partitions. Leave refine_factor unset: + # returning IDs/distances must not fetch or rerank raw vectors. + search(queries) + latencies, results = [], [] + for query in queries: + started = time.perf_counter() + result = search(query) + latencies.append(time.perf_counter() - started) + results.append(result["_rowid"].to_numpy()) + batch_s = [] + for _ in range(3): + started = time.perf_counter() + batch = search(queries) + batch_s.append(time.perf_counter() - started) + query_index = batch["query_index"].to_numpy() + row_ids = batch["_rowid"].to_numpy() + batch_results = [row_ids[query_index == i] for i in range(args.nq)] + print(json.dumps({ + "engine": "lance", "version": lance.__version__, + "machine": platform.machine(), "repeat": repeat, + "base": str(args.base), "n": len(base), "d": base.shape[1], + "nq": args.nq, "k": args.k, "nlist": args.nlist, + "train_n": args.train_n, "threads": args.threads, + "nprobe": nprobe, "query_parallelism": parallelism, + "data_write_s": data_write_s, "index_build_s": index_build_s, + "index_bytes": index_bytes, "recall": recall(results), + "p50_ms": float(np.percentile(latencies, 50) * 1000), + "p95_ms": float(np.percentile(latencies, 95) * 1000), + "sequential_qps": args.nq / sum(latencies), + "batch_s": batch_s, "batch_qps": args.nq / float(np.median(batch_s)), + "batch_recall": recall(batch_results), "index_path": str(path), + }), flush=True) + + +if __name__ == "__main__": + main() From 672d277a30636b449a926531b943a6320522d0ab Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 6 Sep 2026 20:16:47 +0800 Subject: [PATCH 2/3] docs: focus IVF-SQ benchmarks on implementation improvements --- docs/benchmarks/ivfsq-lance-20260906.json | 1372 --------------------- docs/development.html | 36 +- docs/index.html | 26 +- docs/ivf-sq-performance.md | 208 ---- docs/ivf-sq.html | 21 +- docs/releases.html | 2 +- tools/README.md | 13 - tools/benchmark_ivfsq_reader.py | 105 -- tools/benchmark_lance_ivfsq.py | 148 --- 9 files changed, 25 insertions(+), 1906 deletions(-) delete mode 100644 docs/benchmarks/ivfsq-lance-20260906.json delete mode 100644 docs/ivf-sq-performance.md delete mode 100644 tools/benchmark_ivfsq_reader.py delete mode 100644 tools/benchmark_lance_ivfsq.py diff --git a/docs/benchmarks/ivfsq-lance-20260906.json b/docs/benchmarks/ivfsq-lance-20260906.json deleted file mode 100644 index 145e336..0000000 --- a/docs/benchmarks/ivfsq-lance-20260906.json +++ /dev/null @@ -1,1372 +0,0 @@ -{ - "baseline_commit": "8dcabf208c99707eab3938af0bcc40530fdb4cad", - "date": "2026-09-06", - "parameters": { - "nlist": 1024, - "nprobe": 64, - "k": 10, - "nq": 1000, - "train_n": 65536, - "threads": 8, - "repetitions": 3, - "lance_version": "11.0.0", - "paimon_memory_budget_bytes": 4294967296, - "lance_index_cache_bytes": 1073741824, - "lance_training": "native defaults; sample_rate=64", - "lance_cache_parameter": "index_cache_size_bytes" - }, - "paimon_rust": [ - { - "dataset": "sift", - "implementation": "baseline", - "repeat": 1, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 942, - "train_ms": 266, - "add_ms": 549, - "write_ms": 125, - "peak_rss_bytes": 830668800, - "file_bytes": 131359859, - "recall_at_10": 0.8626, - "first_query_us": 903, - "p50_query_us": 709, - "p95_query_us": 845, - "sequential_qps": 1389.47, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 8790731156, - "batch_ms": 158, - "batch_qps": 6294.27, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "baseline", - "repeat": 2, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 934, - "train_ms": 256, - "add_ms": 554, - "write_ms": 122, - "peak_rss_bytes": 834748416, - "file_bytes": 131359859, - "recall_at_10": 0.8626, - "first_query_us": 889, - "p50_query_us": 699, - "p95_query_us": 822, - "sequential_qps": 1418.89, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 8790731156, - "batch_ms": 155, - "batch_qps": 6416.66, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "baseline", - "repeat": 3, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 928, - "train_ms": 258, - "add_ms": 544, - "write_ms": 125, - "peak_rss_bytes": 836927488, - "file_bytes": 131359859, - "recall_at_10": 0.8626, - "first_query_us": 861, - "p50_query_us": 695, - "p95_query_us": 840, - "sequential_qps": 1412.83, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 8790731156, - "batch_ms": 154, - "batch_qps": 6459.88, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "optimized_uncached", - "repeat": 1, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 850, - "train_ms": 251, - "add_ms": 526, - "write_ms": 72, - "peak_rss_bytes": 817119232, - "file_bytes": 131359859, - "recall_at_10": 0.9811, - "first_query_us": 956, - "p50_query_us": 647, - "p95_query_us": 775, - "sequential_qps": 1529.28, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 8790731156, - "batch_ms": 108, - "batch_qps": 9175.67, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "optimized_uncached", - "repeat": 2, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 863, - "train_ms": 251, - "add_ms": 529, - "write_ms": 82, - "peak_rss_bytes": 823115776, - "file_bytes": 131359859, - "recall_at_10": 0.9811, - "first_query_us": 858, - "p50_query_us": 636, - "p95_query_us": 756, - "sequential_qps": 1557.09, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 8790731156, - "batch_ms": 111, - "batch_qps": 9001.14, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "optimized_uncached", - "repeat": 3, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 839, - "train_ms": 250, - "add_ms": 519, - "write_ms": 68, - "peak_rss_bytes": 815874048, - "file_bytes": 131359859, - "recall_at_10": 0.9811, - "first_query_us": 873, - "p50_query_us": 630, - "p95_query_us": 761, - "sequential_qps": 1560.51, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 8790731156, - "batch_ms": 104, - "batch_qps": 9527.79, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "optimized_cached", - "repeat": 1, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 886, - "train_ms": 253, - "add_ms": 538, - "write_ms": 93, - "peak_rss_bytes": 819658752, - "file_bytes": 131359859, - "recall_at_10": 0.9811, - "first_query_us": 895, - "p50_query_us": 193, - "p95_query_us": 300, - "sequential_qps": 4835.36, - "sequential_pread_rounds": 98, - "sequential_pread_bytes": 120400214, - "batch_ms": 110, - "batch_qps": 9046.32, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "optimized_cached", - "repeat": 2, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 879, - "train_ms": 251, - "add_ms": 540, - "write_ms": 87, - "peak_rss_bytes": 828325888, - "file_bytes": 131359859, - "recall_at_10": 0.9811, - "first_query_us": 903, - "p50_query_us": 194, - "p95_query_us": 298, - "sequential_qps": 4818.56, - "sequential_pread_rounds": 98, - "sequential_pread_bytes": 120400214, - "batch_ms": 111, - "batch_qps": 8986.74, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "sift", - "implementation": "optimized_cached", - "repeat": 3, - "n": 1000000, - "d": 128, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 902, - "train_ms": 252, - "add_ms": 521, - "write_ms": 106, - "peak_rss_bytes": 825360384, - "file_bytes": 131359859, - "recall_at_10": 0.9811, - "first_query_us": 936, - "p50_query_us": 204, - "p95_query_us": 275, - "sequential_qps": 4665.87, - "sequential_pread_rounds": 98, - "sequential_pread_bytes": 120400214, - "batch_ms": 122, - "batch_qps": 8153.26, - "batch_pread_rounds": 2, - "batch_pread_bytes": 129759867 - }, - { - "dataset": "glove", - "implementation": "baseline", - "repeat": 1, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 849, - "train_ms": 190, - "add_ms": 539, - "write_ms": 118, - "peak_rss_bytes": 756957184, - "file_bytes": 121822316, - "recall_at_10": 0.8036, - "first_query_us": 893, - "p50_query_us": 613, - "p95_query_us": 722, - "sequential_qps": 1602.82, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 7329440423, - "batch_ms": 141, - "batch_qps": 7042.59, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "baseline", - "repeat": 2, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 873, - "train_ms": 189, - "add_ms": 563, - "write_ms": 119, - "peak_rss_bytes": 763068416, - "file_bytes": 121822316, - "recall_at_10": 0.8036, - "first_query_us": 741, - "p50_query_us": 622, - "p95_query_us": 743, - "sequential_qps": 1569.14, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 7329440423, - "batch_ms": 143, - "batch_qps": 6988.29, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "baseline", - "repeat": 3, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 832, - "train_ms": 190, - "add_ms": 533, - "write_ms": 108, - "peak_rss_bytes": 762527744, - "file_bytes": 121822316, - "recall_at_10": 0.8036, - "first_query_us": 743, - "p50_query_us": 616, - "p95_query_us": 739, - "sequential_qps": 1580.78, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 7329440423, - "batch_ms": 144, - "batch_qps": 6939.38, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "optimized_uncached", - "repeat": 1, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 786, - "train_ms": 183, - "add_ms": 534, - "write_ms": 68, - "peak_rss_bytes": 773079040, - "file_bytes": 121822316, - "recall_at_10": 0.876, - "first_query_us": 755, - "p50_query_us": 564, - "p95_query_us": 681, - "sequential_qps": 1733.36, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 7329440423, - "batch_ms": 97, - "batch_qps": 10224.32, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "optimized_uncached", - "repeat": 2, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 781, - "train_ms": 183, - "add_ms": 526, - "write_ms": 69, - "peak_rss_bytes": 764608512, - "file_bytes": 121822316, - "recall_at_10": 0.876, - "first_query_us": 814, - "p50_query_us": 575, - "p95_query_us": 682, - "sequential_qps": 1702.6, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 7329440423, - "batch_ms": 98, - "batch_qps": 10102.09, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "optimized_uncached", - "repeat": 3, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 760, - "train_ms": 183, - "add_ms": 509, - "write_ms": 66, - "peak_rss_bytes": 771784704, - "file_bytes": 121822316, - "recall_at_10": 0.876, - "first_query_us": 688, - "p50_query_us": 571, - "p95_query_us": 673, - "sequential_qps": 1729.91, - "sequential_pread_rounds": 1000, - "sequential_pread_bytes": 7329440423, - "batch_ms": 93, - "batch_qps": 10697.93, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "optimized_cached", - "repeat": 1, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 803, - "train_ms": 187, - "add_ms": 526, - "write_ms": 90, - "peak_rss_bytes": 768311296, - "file_bytes": 121822316, - "recall_at_10": 0.876, - "first_query_us": 656, - "p50_query_us": 179, - "p95_query_us": 282, - "sequential_qps": 5208.74, - "sequential_pread_rounds": 112, - "sequential_pread_bytes": 113643808, - "batch_ms": 99, - "batch_qps": 10009.12, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "optimized_cached", - "repeat": 2, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 797, - "train_ms": 190, - "add_ms": 517, - "write_ms": 89, - "peak_rss_bytes": 772472832, - "file_bytes": 121822316, - "recall_at_10": 0.876, - "first_query_us": 665, - "p50_query_us": 179, - "p95_query_us": 282, - "sequential_qps": 5174.53, - "sequential_pread_rounds": 112, - "sequential_pread_bytes": 113643808, - "batch_ms": 97, - "batch_qps": 10296.44, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "glove", - "implementation": "optimized_cached", - "repeat": 3, - "n": 1183514, - "d": 100, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 793, - "train_ms": 185, - "add_ms": 516, - "write_ms": 91, - "peak_rss_bytes": 767737856, - "file_bytes": 121822316, - "recall_at_10": 0.876, - "first_query_us": 739, - "p50_query_us": 187, - "p95_query_us": 273, - "sequential_qps": 5024.33, - "sequential_pread_rounds": 112, - "sequential_pread_bytes": 113643808, - "batch_ms": 100, - "batch_qps": 9986.07, - "batch_pread_rounds": 2, - "batch_pread_bytes": 120397102 - }, - { - "dataset": "gist", - "implementation": "baseline", - "repeat": 1, - "n": 1000000, - "d": 960, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 6485, - "train_ms": 1829, - "add_ms": 3718, - "write_ms": 935, - "peak_rss_bytes": 5446975488, - "file_bytes": 973626471, - "recall_at_10": 0.8576, - "first_query_us": 4444, - "p50_query_us": 4296, - "p95_query_us": 4663, - "sequential_qps": 237.19, - "sequential_pread_rounds": 1892, - "sequential_pread_bytes": 74395897851, - "batch_ms": 1026, - "batch_qps": 974.58, - "batch_pread_rounds": 15, - "batch_pread_bytes": 961230121 - }, - { - "dataset": "gist", - "implementation": "optimized_cached", - "repeat": 1, - "n": 1000000, - "d": 960, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 5712, - "train_ms": 1696, - "add_ms": 3327, - "write_ms": 688, - "peak_rss_bytes": 5239685120, - "file_bytes": 973626471, - "recall_at_10": 0.94, - "first_query_us": 3771, - "p50_query_us": 1431, - "p95_query_us": 1788, - "sequential_qps": 687.92, - "sequential_pread_rounds": 162, - "sequential_pread_bytes": 891296688, - "batch_ms": 921, - "batch_qps": 1084.73, - "batch_pread_rounds": 15, - "batch_pread_bytes": 961230121 - }, - { - "dataset": "gist", - "implementation": "baseline", - "repeat": 2, - "n": 1000000, - "d": 960, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 6191, - "train_ms": 1727, - "add_ms": 3671, - "write_ms": 792, - "peak_rss_bytes": 5452349440, - "file_bytes": 973626471, - "recall_at_10": 0.8576, - "first_query_us": 4959, - "p50_query_us": 4060, - "p95_query_us": 4386, - "sequential_qps": 247.87, - "sequential_pread_rounds": 1892, - "sequential_pread_bytes": 74395897851, - "batch_ms": 1112, - "batch_qps": 899.2, - "batch_pread_rounds": 15, - "batch_pread_bytes": 961230121 - }, - { - "dataset": "gist", - "implementation": "optimized_cached", - "repeat": 2, - "n": 1000000, - "d": 960, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 5850, - "train_ms": 1701, - "add_ms": 3661, - "write_ms": 486, - "peak_rss_bytes": 5228003328, - "file_bytes": 973626471, - "recall_at_10": 0.94, - "first_query_us": 4340, - "p50_query_us": 1485, - "p95_query_us": 1828, - "sequential_qps": 665.58, - "sequential_pread_rounds": 162, - "sequential_pread_bytes": 891296688, - "batch_ms": 1007, - "batch_qps": 992.22, - "batch_pread_rounds": 15, - "batch_pread_bytes": 961230121 - }, - { - "dataset": "gist", - "implementation": "baseline", - "repeat": 3, - "n": 1000000, - "d": 960, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 6404, - "train_ms": 1741, - "add_ms": 3705, - "write_ms": 957, - "peak_rss_bytes": 5452398592, - "file_bytes": 973626471, - "recall_at_10": 0.8576, - "first_query_us": 4403, - "p50_query_us": 3970, - "p95_query_us": 4319, - "sequential_qps": 253.2, - "sequential_pread_rounds": 1892, - "sequential_pread_bytes": 74395897851, - "batch_ms": 1113, - "batch_qps": 897.67, - "batch_pread_rounds": 15, - "batch_pread_bytes": 961230121 - }, - { - "dataset": "gist", - "implementation": "optimized_cached", - "repeat": 3, - "n": 1000000, - "d": 960, - "train_n": 65536, - "nq": 1000, - "nlist": 1024, - "nprobe": 64, - "build_ms": 5866, - "train_ms": 1690, - "add_ms": 3613, - "write_ms": 563, - "peak_rss_bytes": 5227593728, - "file_bytes": 973626471, - "recall_at_10": 0.94, - "first_query_us": 4096, - "p50_query_us": 1466, - "p95_query_us": 1936, - "sequential_qps": 651.23, - "sequential_pread_rounds": 162, - "sequential_pread_bytes": 891296688, - "batch_ms": 1001, - "batch_qps": 998.4, - "batch_pread_rounds": 15, - "batch_pread_bytes": 961230121 - } - ], - "paimon_python": [ - { - "corpus": "sift", - "recall": 0.9812, - "p95_ms": 0.25953779331757676, - "qps": 4905.160454190915, - "batch_qps": 10024.995622783992, - "batch_s": [ - 0.09976404199551325, - 0.09975066699553281, - 0.09955183300189674 - ], - "repeat": 1 - }, - { - "corpus": "sift", - "recall": 0.9812, - "p95_ms": 0.26068329170811916, - "qps": 4853.263991636916, - "batch_qps": 9705.435943827024, - "batch_s": [ - 0.10371475000283681, - 0.10281208300148137, - 0.1030350419896422 - ], - "repeat": 2 - }, - { - "corpus": "sift", - "recall": 0.9812, - "p95_ms": 0.25635589117882773, - "qps": 4650.050805561705, - "batch_qps": 9926.987010163688, - "batch_s": [ - 0.10073550000379328, - 0.10022358300921042, - 0.10198312500142492 - ], - "repeat": 3 - }, - { - "corpus": "glove", - "recall": 0.876, - "p95_ms": 0.24610684049548576, - "qps": 5230.060561814994, - "batch_qps": 10923.012825206137, - "batch_s": [ - 0.09154983299958985, - 0.09192229199106805, - 0.09026929199171718 - ], - "repeat": 1 - }, - { - "corpus": "glove", - "recall": 0.876, - "p95_ms": 0.24004789374885144, - "qps": 5253.1115954928655, - "batch_qps": 11057.981564664737, - "batch_s": [ - 0.09043241699691862, - 0.09043366598780267, - 0.09005095799511764 - ], - "repeat": 2 - }, - { - "corpus": "glove", - "recall": 0.876, - "p95_ms": 0.23775805893819774, - "qps": 4362.8639692278075, - "batch_qps": 11051.921929271042, - "batch_s": [ - 0.09028120899165515, - 0.09048199999961071, - 0.10996950000117067 - ], - "repeat": 3 - }, - { - "corpus": "gist", - "recall": 0.9399, - "p95_ms": 1.6917080560233444, - "qps": 708.0206188151822, - "batch_qps": 1141.0011694863388, - "batch_s": [ - 0.8709349999990081, - 0.876423291003448, - 0.8788634160009678 - ], - "repeat": 1, - "batch_recall": 0.94 - }, - { - "corpus": "gist", - "recall": 0.9399, - "p95_ms": 1.7692583467578515, - "qps": 679.2866663514386, - "batch_qps": 981.6139216649766, - "batch_s": [ - 0.9849646670045331, - 1.0199584999936633, - 1.0187304580031196 - ], - "repeat": 2, - "batch_recall": 0.94 - }, - { - "corpus": "gist", - "recall": 0.9399, - "p95_ms": 1.8572527049400378, - "qps": 667.4037218128531, - "batch_qps": 986.4969530139211, - "batch_s": [ - 1.0136878750054166, - 1.0033612500119489, - 1.0273174589965492 - ], - "repeat": 3, - "batch_recall": 0.94 - } - ], - "lance": [ - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 0, - "n": 1000000, - "d": 128, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 0.31032995801069774, - "index_build_s": 3.6129824169911444, - "index_bytes": 131369309, - "recall": 0.9797, - "p50_ms": 1.0014379950007424, - "p95_ms": 1.2794169888366012, - "sequential_qps": 989.3228202686936, - "batch_s": [ - 0.2415808749938151, - 0.23636166600044817, - 0.2391535839997232 - ], - "batch_qps": 4181.413396678, - "batch_recall": 0.9797, - "dataset": "sift" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 0, - "n": 1000000, - "d": 128, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 0.31032995801069774, - "index_build_s": 3.6129824169911444, - "index_bytes": 131369309, - "recall": 0.9793, - "p50_ms": 0.8949789917096496, - "p95_ms": 1.0652399017999412, - "sequential_qps": 1102.8434043752866, - "batch_s": [ - 0.3082639579952229, - 0.31552245799684897, - 0.30906400000094436 - ], - "batch_qps": 3235.5758030600277, - "batch_recall": 0.9793, - "dataset": "sift" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 1, - "n": 1000000, - "d": 128, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 0.5087806249939604, - "index_build_s": 3.1083727079967503, - "index_bytes": 131369053, - "recall": 0.9771, - "p50_ms": 1.2891464939457364, - "p95_ms": 1.631252094375668, - "sequential_qps": 844.603676273628, - "batch_s": [ - 0.2487101669976255, - 0.25258970899449196, - 0.2553312910022214 - ], - "batch_qps": 3958.9894773654705, - "batch_recall": 0.9771, - "dataset": "sift" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 1, - "n": 1000000, - "d": 128, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 0.5087806249939604, - "index_build_s": 3.1083727079967503, - "index_bytes": 131369053, - "recall": 0.9772, - "p50_ms": 1.0143955005332828, - "p95_ms": 5.934893446101337, - "sequential_qps": 495.67022593381444, - "batch_s": [ - 0.38212937500793487, - 0.5658738329948392, - 0.46950991699122824 - ], - "batch_qps": 2129.8804643112207, - "batch_recall": 0.9772, - "dataset": "sift" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 2, - "n": 1000000, - "d": 128, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 0.5426220420049503, - "index_build_s": 6.131838665998657, - "index_bytes": 131368925, - "recall": 0.9775, - "p50_ms": 1.119521002692636, - "p95_ms": 3.3385020491550677, - "sequential_qps": 591.3198932006269, - "batch_s": [ - 0.22399091700208373, - 0.21637758301221766, - 0.21978304100048263 - ], - "batch_qps": 4549.941594437235, - "batch_recall": 0.9775, - "dataset": "sift" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 2, - "n": 1000000, - "d": 128, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 0.5426220420049503, - "index_build_s": 6.131838665998657, - "index_bytes": 131368925, - "recall": 0.977, - "p50_ms": 0.9637295006541535, - "p95_ms": 1.0986836452502757, - "sequential_qps": 1017.894235910497, - "batch_s": [ - 0.27645879100600723, - 0.2729735000029905, - 0.26712979200237896 - ], - "batch_qps": 3663.3592637711895, - "batch_recall": 0.977, - "dataset": "sift" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 0, - "n": 1183514, - "d": 100, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 0.4582100830011768, - "index_build_s": 2.8858480420021806, - "index_bytes": 123788044, - "recall": 0.784, - "p50_ms": 1.1995829991064966, - "p95_ms": 1.499956246698275, - "sequential_qps": 826.7088574803406, - "batch_s": [ - 0.23057912499643862, - 0.23458475001098122, - 0.23765391600318253 - ], - "batch_qps": 4262.851698386995, - "batch_recall": 0.784, - "dataset": "glove" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 0, - "n": 1183514, - "d": 100, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 0.4582100830011768, - "index_build_s": 2.8858480420021806, - "index_bytes": 123788044, - "recall": 0.7841, - "p50_ms": 0.9254164979211055, - "p95_ms": 1.0633017438522072, - "sequential_qps": 1069.245148811762, - "batch_s": [ - 0.3005618330062134, - 0.29484195800614543, - 0.28533616701315623 - ], - "batch_qps": 3391.6475347079227, - "batch_recall": 0.7841, - "dataset": "glove" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 1, - "n": 1183514, - "d": 100, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 0.42901387499296106, - "index_build_s": 2.789812582996092, - "index_bytes": 123794125, - "recall": 0.7845, - "p50_ms": 1.251395995495841, - "p95_ms": 1.5928913599054795, - "sequential_qps": 796.3681045469514, - "batch_s": [ - 0.23582358300336637, - 0.23455374999321066, - 0.24161516700405627 - ], - "batch_qps": 4240.458003666771, - "batch_recall": 0.7845, - "dataset": "glove" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 1, - "n": 1183514, - "d": 100, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 0.42901387499296106, - "index_build_s": 2.789812582996092, - "index_bytes": 123794125, - "recall": 0.7845, - "p50_ms": 0.9430415011593141, - "p95_ms": 1.0890444049437062, - "sequential_qps": 1050.3114299475167, - "batch_s": [ - 0.32420116700814106, - 0.2876453750068322, - 0.30344629200408235 - ], - "batch_qps": 3295.476090334123, - "batch_recall": 0.7845, - "dataset": "glove" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 2, - "n": 1183514, - "d": 100, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 0.35886495900922455, - "index_build_s": 2.755436583989649, - "index_bytes": 123799052, - "recall": 0.7845, - "p50_ms": 1.228708504640963, - "p95_ms": 1.5429267579747827, - "sequential_qps": 804.5836861276184, - "batch_s": [ - 0.23650141699181404, - 0.24138320799102075, - 0.2385975840006722 - ], - "batch_qps": 4191.157275076107, - "batch_recall": 0.7845, - "dataset": "glove" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 2, - "n": 1183514, - "d": 100, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 0.35886495900922455, - "index_build_s": 2.755436583989649, - "index_bytes": 123799052, - "recall": 0.7843, - "p50_ms": 1.055813001585193, - "p95_ms": 1.403788961761165, - "sequential_qps": 872.8329078474785, - "batch_s": [ - 0.30117095899186097, - 0.29349999999976717, - 0.3128447079943726 - ], - "batch_qps": 3320.373263568964, - "batch_recall": 0.7843, - "dataset": "glove" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 0, - "n": 1000000, - "d": 960, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 4.566240332991583, - "index_build_s": 18.651229708993924, - "index_bytes": 966467340, - "recall": 0.9249, - "p50_ms": 3.5183125000912696, - "p95_ms": 4.213342302682577, - "sequential_qps": 279.86572814988415, - "batch_s": [ - 0.7250397089956095, - 0.7115559579979163, - 0.7171322499925736 - ], - "batch_qps": 1394.4429357490974, - "batch_recall": 0.9249, - "dataset": "gist" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 0, - "n": 1000000, - "d": 960, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 4.566240332991583, - "index_build_s": 18.651229708993924, - "index_bytes": 966467340, - "recall": 0.9248, - "p50_ms": 1.60639600653667, - "p95_ms": 2.045174988597864, - "sequential_qps": 605.6153423224837, - "batch_s": [ - 0.7599140410020482, - 0.7975927080115071, - 0.7592186659894651 - ], - "batch_qps": 1315.9383114981879, - "batch_recall": 0.9248, - "dataset": "gist" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 1, - "n": 1000000, - "d": 960, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 3.40062129200669, - "index_build_s": 18.463206583997817, - "index_bytes": 966466660, - "recall": 0.9326, - "p50_ms": 3.4582089938339777, - "p95_ms": 4.417935459787259, - "sequential_qps": 282.8399689006948, - "batch_s": [ - 0.7494371249922551, - 0.7551345410029171, - 0.7434802909992868 - ], - "batch_qps": 1334.3347515781718, - "batch_recall": 0.9326, - "dataset": "gist" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 1, - "n": 1000000, - "d": 960, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 3.40062129200669, - "index_build_s": 18.463206583997817, - "index_bytes": 966466660, - "recall": 0.9325, - "p50_ms": 1.558103998831939, - "p95_ms": 1.8234190487419255, - "sequential_qps": 638.5420358786371, - "batch_s": [ - 0.8040630000032252, - 0.8117980000097305, - 0.8028900000063004 - ], - "batch_qps": 1243.6836416997037, - "batch_recall": 0.9325, - "dataset": "gist" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 2, - "n": 1000000, - "d": 960, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 0, - "data_write_s": 2.69620574999135, - "index_build_s": 19.185824375002994, - "index_bytes": 966467596, - "recall": 0.9249, - "p50_ms": 3.6111040026298724, - "p95_ms": 4.190530948108062, - "sequential_qps": 277.0624360371063, - "batch_s": [ - 0.7597572090016911, - 0.772940082999412, - 0.7539892079948913 - ], - "batch_qps": 1316.2099525373167, - "batch_recall": 0.9249, - "dataset": "gist" - }, - { - "engine": "lance", - "version": "11.0.0", - "machine": "arm64", - "repeat": 2, - "n": 1000000, - "d": 960, - "nq": 1000, - "k": 10, - "nlist": 1024, - "train_n": 65536, - "threads": 8, - "nprobe": 64, - "query_parallelism": 8, - "data_write_s": 2.69620574999135, - "index_build_s": 19.185824375002994, - "index_bytes": 966467596, - "recall": 0.9249, - "p50_ms": 1.60087500262307, - "p95_ms": 1.8618172958667856, - "sequential_qps": 621.3550108671919, - "batch_s": [ - 0.8047680419986136, - 0.8261880420031957, - 0.8697780829970725 - ], - "batch_qps": 1210.378205880801, - "batch_recall": 0.9249, - "dataset": "gist" - } - ], - "hardware": { - "cpu": "Apple M4 Pro", - "logical_cpus": 12, - "ram_bytes": 51539607552 - }, - "dataset_sources": { - "gist": { - "url": "https://ann-benchmarks.com/gist-960-euclidean.hdf5", - "content_length": 3844648288, - "etag": "34da1d8a80764582ee4b0c0839b7c32a-459", - "n": 1000000, - "d": 960, - "query_count": 1000, - "normalized": false - } - } -} diff --git a/docs/development.html b/docs/development.html index fae82fc..fc3e5f7 100644 --- a/docs/development.html +++ b/docs/development.html @@ -26,7 +26,7 @@

Build · test · measure

Development and benchmarks

Run the standard Rust checks, exercise the C/C++, Java, and Python integrations, and measure ANN and filtered-query behavior with reproducible workloads.

Cargo workspaceCMake smoke testsMaven / pytestCriterion benchmarks
- +

Repository layout

@@ -106,29 +106,17 @@

Public SIFT1M, GIST1M, and GloVe-100 data

The benchmark executes three serving cases by default. local_ssd_warm_cache uses the real index file with no artificial delay. remote_cache_2ms adds a fixed 2 ms delay per pread call while reading all ranges in that call concurrently. object_store_20ms uses the ObjectStore read plan: open/optimization and sequential-query latency are the measured CPU/I/O time plus 20 ms for every observed dependent read round, avoiding hours of idle wall-clock sleep while retaining the exact fixed-latency model; batch QPS still injects the real 20 ms sleep so concurrent-client overlap is measured rather than inferred. These models do not include bandwidth or operating-system cache misses. DiskANN uses a dedicated range-I/O pool separate from Rayon query workers, preventing a saturated batch from starving nested positional reads; IVF retains the shared compute pool because its batch read scheduling is not nested inside per-query sessions. The unified IVF Reader reuses the 64-byte type-dispatch header and loads resident metadata in one further contiguous read, reducing initialization from three rounds to two without changing any format. DiskANN is opened explicitly as LocalStorage, RemoteStorage, or ObjectStore with the default 4 GiB Reader budget, which is automatically partitioned into resident state and caches. The sequential sweep and batch call use separate opened-and-optimized Readers, preventing all benchmark queries from prewarming their own batch measurement. IVF-SQ now uses the default reader budget for a partition cache: sequential queries can reuse prior partitions, while the first batch includes payload reads and cache insertion. Simulated remote delay applies only to actual positional-read calls; cache hits issue none. CSV output records optimization rounds, ranges, and bytes separately from query I/O. Use ANN_STORAGE_CASES for a focused rerun against a retained index. See the recorded public-corpus results and interpretation.

-
-

Reproduce the IVF-SQ / Lance comparison

-

The September 2026 results compare SIFT1M, GIST1M, and normalized GloVe-100 with Lance 11.0.0. Use the same converted public base, query, and ground-truth files for both engines. Install numpy, h5py, pyarrow, and pylance==11.0.0 in a temporary Python environment. Run from the repository root, one benchmark at a time, with no concurrent compilation.

-
    -
  1. Convert the public data using the commands above, retaining the first 1,000 held-out queries. Use --normalize-l2 for GloVe before either engine reads it.
    2. -
  2. Build with cargo bench -p paimon-vindex-core --bench ann_bench --no-run and cargo build --release -p paimon-vindex-ffi. For Paimon, set ANN_INDEXES=IVF_SQ, RAYON_NUM_THREADS=8, ANN_TRAIN_N=65536, ANN_NLIST=1024, ANN_NPROBE=64, ANN_K=10, and ANN_STORAGE_CASES=local_ssd_warm_cache. Set the three public-data paths, ANN_KEEP_INDEXES=1, and an ANN_OUTPUT_DIR; run three separate build repetitions.
    3. -
  3. Run benchmark_lance_ivfsq.py on the same inputs. It reports source-dataset writing separately from index construction, tests both query scheduling modes, and warms the probed partitions before timing.
    4. -
  4. Run benchmark_ivfsq_reader.py on a retained ivf_sq.index. It uses the public Python reader with ordinary positional file-read callbacks and warms partitions through batch search before timing.
    5. -
-
Shell · after exporting the public ANN paths and setting OUT and INDEX
python tools/benchmark_lance_ivfsq.py \
-  --base "$ANN_BASE_FVECS" --queries "$ANN_QUERY_FVECS" \
-  --ground-truth "$ANN_GROUND_TRUTH_IVECS" --output-dir "$OUT/lance" \
-  --threads 8 --train-n 65536 --nlist 1024 --nprobe 64 --nq 1000 --k 10 \
-  --query-parallelism 0 8 --cache-bytes 1073741824 --repeats 3
-
-PYTHONPATH=python PAIMON_VINDEX_LIB_PATH="$PWD/target/release/libpaimon_vindex_ffi.dylib" \
-python tools/benchmark_ivfsq_reader.py --index "$INDEX" \
-  --queries "$ANN_QUERY_FVECS" --ground-truth "$ANN_GROUND_TRUTH_IVECS" \
-  --threads 8 --nprobe 64 --nq 1000 --k 10 \
-  --memory-budget-bytes 4294967296 --repeats 3
-

OUT is the benchmark output directory and INDEX is the file retained under ANN_OUTPUT_DIR/<pid>/ivf_sq.index. Use the .so library on Linux. The full command sequence includes data conversion and Paimon build settings; raw JSON preserves all repetitions.

-
Keep the measurement boundaries explicitThe report's query table uses warmed Python readers for both engines; Rust ann_bench times its batch on a fresh reader, including reads and cache insertion. IVF-SQ's warmup_queries does not load partitions: use actual searches. A reader budget of zero measures the uncached SQ path, not a cold operating-system cache.
-

Paimon's budget is 4 GiB and Lance's index cache is 1 GiB; both fit each measured index. Lance uses the byte-count option index_cache_size_bytes. Its deprecated index_cache_size counts entries and must not be used as a byte budget. Match training sample counts while allowing each engine its native training algorithm and iteration policy. Report recall alongside latency, exclude Lance source-dataset writing from index-build comparisons, and state which scheduling mode supplies each metric.

+
+

Reproduce the IVF-SQ benchmark

+

The September 2026 results compare the preceding implementation at 8dcabf2 with the optimized IVF-SQ implementation on SIFT1M, GIST1M, and normalized GloVe-100. Convert the public data as described above, retaining the first 1,000 held-out queries and using --normalize-l2 for GloVe. Run from the repository root, one benchmark at a time, with no concurrent compilation.

+
Shell · after exporting the three public ANN file paths
cargo bench -p paimon-vindex-core --bench ann_bench --no-run
+RAYON_NUM_THREADS=8 ANN_INDEXES=IVF_SQ ANN_TRAIN_N=65536 \
+ANN_NLIST=1024 ANN_NPROBE=64 ANN_K=10 \
+ANN_STORAGE_CASES=local_ssd_warm_cache \
+ANN_OUTPUT_DIR=/path/on/target/ssd \
+cargo bench -p paimon-vindex-core --bench ann_bench
+

Set ANN_BASE_FVECS, ANN_QUERY_FVECS, and ANN_GROUND_TRUTH_IVECS to the converted files. Repeat each corpus three times in separate processes and report the median of each metric. Use the same data and settings for baseline and current builds. The recorded runs used Rust 1.95 release builds on Apple M4 Pro (12 CPU cores, 48 GiB RAM). Set ANN_KEEP_INDEXES=1 to retain each generated ANN_OUTPUT_DIR/<pid>/ivf_sq.index for compatibility checks.

+
Measurement boundariesBuild time includes training, encoding, and serialization. Native ann_bench uses a 4 GiB reader budget: sequential queries can reuse partitions loaded earlier in the sweep; batch timing uses a separate fresh reader and includes payload reads and cache insertion. The baseline reader does not cache SQ partitions. The local results use a warm filesystem cache and do not measure cold storage or object-store behavior.
diff --git a/docs/index.html b/docs/index.html index 58dc7f3..c405008 100644 --- a/docs/index.html +++ b/docs/index.html @@ -117,36 +117,16 @@

Core differences

-
-
-

6 September 2026: IVF-SQ versus Lance

-

Three-run medians on public SIFT1M, GIST1M, and GloVe-100: Apple M4 Pro, eight workers, 1,024 partitions, 64 probes, Top-10, 65,536 training rows, and 1,000 held-out queries. Both engines use the same input files and native training defaults; GloVe is normalized before either engine reads it.

-
-
- - - - - - -
CorpusEngineIndex buildPython P95Python batch QPSRecall@10
SIFT1MPaimon IVF-SQ0.886 s0.260 ms9,9270.9812
SIFT1MLance 11.0.03.613 s1.099 ms4,1810.9772–0.9775
GIST1MPaimon IVF-SQ5.850 s1.769 ms9860.9399
GIST1MLance 11.0.018.651 s1.862 ms1,3340.9249
GloVe-100Paimon IVF-SQ0.797 s0.240 ms11,0520.8760
GloVe-100Lance 11.0.02.790 s1.089 ms4,2400.7843–0.7845
-

Build includes training through index serialization and excludes Lance source-dataset writing. Queries use both public Python interfaces after warming selected partitions, with no raw-vector refinement. Cache budgets are 4 GiB for Paimon and 1 GiB for Lance; each measured index fits both.

-

Relative to Lance, SIFT1M: 4.08× build speed, 76% lower Python P95, 2.37× batch throughput; GIST1M: 3.19× build speed, 5% lower Python P95, 0.74× batch throughput; GloVe-100: 3.50× build speed, 78% lower Python P95, 2.61× batch throughput.

-
GIST batch throughput still trails LanceIVF-SQ reaches 986 QPS versus Lance's 1,334 QPS, about 26% lower, despite its faster build and higher recall. Its median Python P95 is only 5% lower, and the three-run latency ranges overlap. These results do not establish an across-the-board performance lead.
-

Lance uses its faster scheduling mode separately for each metric: SIFT1M P95 / batch parallelism 8 / 0; GIST1M P95 / batch parallelism 8 / 0; GloVe-100 P95 / batch parallelism 8 / 0. Recall ranges show the two scheduling-mode medians. These measurements do not cover cold storage, object stores, or other architectures.

-

See the full report, reproduction commands, and all recorded runs. The native build and local-storage tables below also contain the refreshed IVF-SQ results.

-
-

Public-corpus benchmark results

IVF-SQ was refreshed on 6 September 2026 across all three corpora. Its build and local-search cells below use three-run native ann_bench medians with eight Rayon workers. The other four index rows retain their July 2026 measurements with 12 workers; these tables record the latest available results for each index, not a new simultaneous five-index ranking.

-

All rows use the same public corpus shapes, 1,000 independent queries, nlist=1024, nprobe=64, and Top-10. Native batch timing includes the first payload reads and cache insertion on a fresh reader, so it differs from the warmed Python comparison above. The remote-model results remain archived separately below.

+

All rows use the same public corpus shapes, 1,000 independent queries, nlist=1024, nprobe=64, and Top-10. Native batch timing includes the first payload reads and cache insertion on a fresh reader. The remote-model results remain archived separately below.

Historical five-index setup and reproduction parameters · July 2026 -

The following records the original 12-worker comparison. For the refreshed eight-worker IVF-SQ configuration, use the current reproduction guide.

+

The following records the original 12-worker comparison. For the refreshed eight-worker IVF-SQ configuration, use the current reproduction guide.

Benchmark setup

This is not the zero-configuration benchmarkRunning cargo bench -p paimon-vindex-core --bench ann_bench without public file paths uses a 20k-vector, 64-dimensional generated smoke workload. Reproducing the results below requires the public files, recorded IVF and DiskANN search settings, a fixed worker count, and an output directory on the storage device being measured. File shape, training count, and multi-index process isolation are automatic.

The real-data run uses the public ANN-Benchmarks SIFT1M, GIST1M, and GloVe-100 files, the first 1,000 independent test queries, and their published Top-100 exact neighbors. Recall@10 compares only the first ten published neighbors. SIFT and GIST contain one million base vectors with 128 and 960 dimensions. GloVe contains 1,183,514 vectors with 100 dimensions and angular ground truth; its base and query vectors are L2-normalized during conversion so the common L2 benchmark produces the same neighbor ordering as cosine. The benchmark supplies 65,536 base vectors to every trainer; DiskANN bounds PQ training memory with a deterministic reservoir of at most 50,000 vectors.

@@ -183,7 +163,7 @@

Native local-storage results

IVF-FLAT0.9937 / 1.88 ms / 8,510 / 33.19 MiB0.9549 / 11.38 ms / 875 / 283.40 MiB0.8832 / 1.40 ms / 9,502 / 27.57 MiB

The refreshed SQ reader reuses decoded partitions within its memory budget. The read column is average payload I/O per sequential query, including cache misses while that sweep warms the reader; it is not the full selected-list size or a cold-cache promise. SQ batch QPS includes the first batch's payload reads on a separate reader. The recorded July rows predate this SQ cache and training change.

-

For IVF-SQ, train / encode-add / serialize medians in SIFT / GIST / GloVe order are 252 / 538 / 93 ms; 1696 / 3613 / 563 ms; 187 / 517 / 90 ms. Stage medians are computed independently and need not sum to the median total. See the same-configuration baseline comparison for the optimization's measured effect.

+

For IVF-SQ, train / encode-add / serialize medians in SIFT / GIST / GloVe order are 252 / 538 / 93 ms; 1696 / 3613 / 563 ms; 187 / 517 / 90 ms. Stage medians are computed independently and need not sum to the median total. See the same-configuration baseline comparison for the optimization's measured effect.

Historical implementation notes and earlier SQ scores · July 2026 diff --git a/docs/ivf-sq-performance.md b/docs/ivf-sq-performance.md deleted file mode 100644 index 3ab3671..0000000 --- a/docs/ivf-sq-performance.md +++ /dev/null @@ -1,208 +0,0 @@ - - -# IVF-SQ versus Lance: September 2026 - -SIFT1M and GloVe improve in build time, Python query latency, batch throughput, -and recall relative to Lance 11.0.0. The GIST1M rerun improves build time and -recall, with a modest 5% lower median Python P95, but **batch throughput remains -26% below Lance**. The goal of beating Lance in every measured workload is not -yet met. These are local measurements with the configuration below. - -The site pages summarize [current results](ivf-sq.html#benchmarks), -[reader options](api.html#reader-options), and [benchmark setup](development.html#ivfsq-lance). - -## Results - -Medians of three runs, Apple M4 Pro (12 CPU cores, 48 GiB RAM), eight workers, -release Rust 1.95.0, 1,024 partitions, 64 probes, Top-10, 65,536 training rows, -and the first 1,000 independent ANN-Benchmarks queries. GloVe base/query -vectors were normalized identically before either engine read them. - -| Corpus | Engine | Index build | Python query P95 | Python batch QPS | Recall@10 | -| --- | --- | ---: | ---: | ---: | ---: | -| SIFT1M, 1M × 128 | Paimon IVF-SQ | 0.886 s | 0.260 ms | 9,927 | 0.9812 | -| SIFT1M | Lance IVF-SQ 11.0.0 | 3.613 s | 1.099 ms | 4,181 | 0.9772–0.9775 | -| GIST1M, 1M × 960 | Paimon IVF-SQ | 5.850 s | 1.769 ms | 986 | 0.9399 | -| GIST1M | Lance IVF-SQ 11.0.0 | 18.651 s | 1.862 ms | 1,334 | 0.9249 | -| GloVe, 1,183,514 × 100 | Paimon IVF-SQ | 0.797 s | 0.240 ms | 11,052 | 0.8760 | -| GloVe | Lance IVF-SQ 11.0.0 | 2.790 s | 1.089 ms | 4,240 | 0.7843–0.7845 | - -In SIFT / GIST / GloVe order, construction is **4.08× / 3.19× / 3.50× as fast** -as Lance, Python P95 is **76% / 5% / 78% lower**, and Python batch throughput -is **2.37× / 0.74× / 2.61×** Lance's. GIST therefore retains a batch-throughput -gap. Its three P95 observations span 1.692–1.857 ms for Paimon and -1.823–2.045 ms for Lance with parallelism 8; the ranges overlap, so the small -median latency advantage should not be treated as a large or universal win. -Lance recall ranges show the medians of the two scheduling modes. Its faster -setting is used separately for each performance metric: partition parallelism -8 for single-query latency and the default 0 for batch throughput. No raw-vector -refinement is requested in either engine. Returned data consists of IDs and -distances; neither benchmark requests the raw vector column. - -The [recorded measurements](benchmarks/ivfsq-lance-20260906.json) include all -three repetitions and both Lance scheduling modes. Both engines use their native training algorithms and iteration policies; -training sample counts match, but learned centroids need not. Lance training is not -bitwise deterministic across builds, so recall varies slightly between runs. -Paimon's Rust and Python result paths can choose different members of a tied -boundary; SIFT and GIST recall differ by 0.0001 between native and Python -single-query entry points. GIST Python batch recall is 0.9400. The GIST rerun -uses three interleaved baseline/current/Python/Lance repetitions on the same day. - -## Changes and baseline - -The baseline is commit `8dcabf208c99707eab3938af0bcc40530fdb4cad`. - -| Rust ann_bench metric | SIFT baseline → current | GIST baseline → current | GloVe baseline → current | -| --- | ---: | ---: | ---: | -| Build | 934 → 886 ms | 6,404 → 5,850 ms | 849 → 797 ms | -| Encode/add | 549 → 538 ms | 3,705 → 3,613 ms | 539 → 517 ms | -| Serialize | 125 → 93 ms | 935 → 563 ms | 118 → 90 ms | -| Query P95 | 840 → 298 µs | 4,386 → 1,828 µs | 739 → 282 µs | -| Batch QPS | 6,417 → 8,987 | 899 → 998 | 6,988 → 10,009 | -| Recall@10 | 0.8626 → 0.9811 | 0.8576 → 0.9400 | 0.8036 → 0.8760 | -| File bytes | 131,359,859 → unchanged | 973,626,471 → unchanged | 121,822,316 → unchanged | - -GIST serialization time falls by 40%, native query P95 is 58% lower, and native -batch throughput is 11% higher than the same eight-worker baseline. Peak process -RSS falls from 5.08 to 4.87 GiB. These improvements do not close the batch gap to -Lance shown above. - -1. **Avoid sparse-partition clipping.** Training previously estimated a separate - minimum and maximum from each partition's often tiny sample. Constant sample - dimensions and narrow extrema clipped unseen residuals. The trainer now pools - per-dimension residual extrema across partitions. Holding centroids fixed, - this alone raised SIFT Recall@10 from about 0.863 to 0.981. Broad pooled bounds - can reduce resolution on corpora with extreme outliers; measure such data - before adopting this training policy. -2. **Fuse encoding and bound training.** Partition-local reductions avoid the - training residual matrix. Encoding computes inverse scales once per partition, - subtracts the centroid in registers, and packs rounded NEON/AVX2 results - directly into the destination. Serialization transposes partitions in parallel - within 16 MiB batches (one oversized partition is processed alone). -3. **Reuse query heaps and prune safely.** Batch scans retain one heap per query. - An L2 block can stop after its first half only if every nonnegative partial - distance already exceeds or equals the current cutoff. Competitive distances - are still evaluated completely. A first partition supplies the cutoff for - parallel single-query scans. A one-query batch uses the single-query path. -4. **Reuse decoded partitions within the existing memory budget.** The unified - reader and bindings use a bounded FIFO cache. Cache hits share immutable - buffers with `Arc`, bypass positional I/O, and avoid decoding IDs again. - Metadata, cache slot/queue storage, and retained payload capacities are - reserved before retaining entries. Filters and distances are never cached. - Oversized streamed partitions bypass the cache. Zero budget disables caching; - required metadata still loads. Direct `IVFSQIndexReader::open` stays uncached; - `open_with_options` enables the cache. - -The SIFT/GloVe uncached optimized intermediate also improved Rust batch -throughput to 9,176 / 10,224 QPS at the new higher recall; this intermediate -was not rerun on GIST. Cache-enabled ann_bench batch -numbers are slightly lower because it opens a fresh reader and times a complete -first batch, including payload reads and cache insertion. The Python comparison -warms the selected partitions before timing both engines. Its native file adapter -uses ordinary `os.pread` callbacks; no special in-memory adapter is used. - -Paimon's configured reader budget is 4 GiB and Lance's index cache is 1 GiB -(explicit `index_cache_size_bytes`, not the deprecated entry-count parameter); -each entire measured index fits within either budget. Neither reserves that -whole amount as payload memory. Paimon's peak process RSS was about -787 / 4,986 / 733 MiB for SIFT / GIST / GloVe, including the benchmark's source data and build phases. Build time includes -training, assignment, encoding and index serialization. Lance additionally needs -a source dataset; its data-writing time is recorded separately and **excluded** -from the comparison above. Paimon build stages use the Rust benchmark; query -latency/throughput in the first table use both public Python interfaces. - -The IVSQ v1 format, flags, row-ID encoding, and golden fixture bytes remain -compatible. Old files benefit from the reader changes immediately. Rebuilding -is required to get the new training bounds; existing files keep their recorded -per-partition quantizers. - -## Reproduce - -Obtain the public SIFT, GIST, and GloVe HDF5 files from -[ANN-Benchmarks](https://github.com/erikbern/ann-benchmarks). Install `numpy`, -`h5py`, `pyarrow`, and `pylance==11.0.0` in a temporary environment. Run each -benchmark separately, without concurrent compilation or other benchmarks. Run from -the repository root, and set `DATA` to the directory containing the downloaded -HDF5 files and `OUT` to the directory for generated indexes before running the -commands below. - -```sh -python tools/convert_ann_benchmarks.py "$DATA/sift-128-euclidean.hdf5" "$DATA/sift" \ - --prefix sift --query-limit 1000 -python tools/convert_ann_benchmarks.py "$DATA/gist-960-euclidean.hdf5" "$DATA/gist" \ - --prefix gist --query-limit 1000 -python tools/convert_ann_benchmarks.py "$DATA/glove-100-angular.hdf5" "$DATA/glove" \ - --prefix glove --query-limit 1000 --normalize-l2 -cargo bench -p paimon-vindex-core --bench ann_bench --no-run -cargo build --release -p paimon-vindex-ffi - -# Repeat for corpus=gist and corpus=glove, and repeat each engine three times. -corpus=sift -export RAYON_NUM_THREADS=8 ANN_INDEXES=IVF_SQ ANN_TRAIN_N=65536 -export ANN_NLIST=1024 ANN_NPROBE=64 ANN_K=10 ANN_STORAGE_CASES=local_ssd_warm_cache -export ANN_BASE_FVECS="$DATA/$corpus/${corpus}_base.fvecs" -export ANN_QUERY_FVECS="$DATA/$corpus/${corpus}_query.fvecs" -export ANN_GROUND_TRUTH_IVECS="$DATA/$corpus/${corpus}_ground_truth.ivecs" -export ANN_KEEP_INDEXES=1 ANN_OUTPUT_DIR="$OUT/paimon-$corpus" -cargo bench -p paimon-vindex-core --bench ann_bench - -python tools/benchmark_lance_ivfsq.py \ - --base "$ANN_BASE_FVECS" --queries "$ANN_QUERY_FVECS" \ - --ground-truth "$ANN_GROUND_TRUTH_IVECS" --output-dir "$OUT/lance-$corpus" \ - --threads 8 --train-n 65536 --nlist 1024 --nprobe 64 --nq 1000 --k 10 \ - --query-parallelism 0 8 --cache-bytes 1073741824 --repeats 3 - -# INDEX is the ivf_sq.index preserved under ANN_OUTPUT_DIR//. -PYTHONPATH=python PAIMON_VINDEX_LIB_PATH="$PWD/target/release/libpaimon_vindex_ffi.dylib" \ -python tools/benchmark_ivfsq_reader.py --index "$INDEX" \ - --queries "$ANN_QUERY_FVECS" --ground-truth "$ANN_GROUND_TRUTH_IVECS" \ - --threads 8 --nprobe 64 --nq 1000 --k 10 \ - --memory-budget-bytes 4294967296 --repeats 3 -``` - -Use `--memory-budget-bytes 0` on the Python reader benchmark to isolate the -uncached scan path; this does not flush the operating-system cache. For IVF-SQ, -`optimize_for_search` and `warmup_queries` initialize metadata only. Replay actual -searches to warm the partition cache, as the Python script does. On Linux, use the `.so` native library instead of `.dylib`. -The Lance script creates a new dataset per repetition and reports the path; -remove those generated benchmark outputs when no longer needed. - -## Verification and limits - -- Workspace tests: 510 passed, 2 intentionally ignored; includes v1 golden fixtures. -- Python bindings: 28 passed. -- The pre-change reader at `8dcabf2` opened newly generated SIFT, GIST, and - GloVe indexes and completed 1,000 single queries plus batch search per corpus. - Recall@10 differed from the new reader by at most 0.0002; this verifies file - compatibility, not identical ordering of every result. -- x86_64 build and SQ tests under Rosetta: 33 passed. Rosetta reported AVX2/FMA - unavailable, so AVX2 was compiled but its runtime kernel needs native x86 CI. -- `cargo fmt`, workspace Clippy with warnings denied, license headers, and diff - whitespace checks passed. -- Added regression coverage for packed-encoding rounding/tails, residual extrema, - sparse/empty partition calibration, L2 cutoff boundaries, single-query and - seeded batch paths, cache reuse, eviction, budget bypass, filtering, and read - failures followed by retries. - -This run does not establish superiority on Linux/x86, cold storage, -object-store latency, or every metric/distribution. GIST batch throughput remains -below Lance. The homepage build/local tables contain the refreshed IVF-SQ rows; -other indexes retain their labeled July measurements, and the historical remote -models and implementation notes remain in collapsible archives. diff --git a/docs/ivf-sq.html b/docs/ivf-sq.html index ddab505..34a52c9 100644 --- a/docs/ivf-sq.html +++ b/docs/ivf-sq.html @@ -67,18 +67,15 @@

I/O and batching

Public benchmarks

-

September 2026: IVF-SQ versus Lance 11.0.0

-

Three-run medians on Apple M4 Pro (12 CPU cores, 48 GiB RAM), with eight workers, nlist=1024, nprobe=64, k=10, 65,536 training rows, and 1,000 held-out public queries. Both engines read the same vectors; GloVe is normalized before either engine loads it. Query measurements use the public Python interfaces with selected partitions warmed and no raw-vector refinement.

-
- - - - - - -
CorpusEngineIndex buildPython P95Python batch QPSRecall@10
SIFT1MPaimon IVF-SQ0.886 s0.260 ms9,9270.9812
SIFT1MLance 11.0.03.613 s1.099 ms4,1810.9772–0.9775
GIST1MPaimon IVF-SQ5.850 s1.769 ms9860.9399
GIST1MLance 11.0.018.651 s1.862 ms1,3340.9249
GloVe-100Paimon IVF-SQ0.797 s0.240 ms11,0520.8760
GloVe-100Lance 11.0.02.790 s1.089 ms4,2400.7843–0.7845
-

In SIFT / GIST / GloVe order, Paimon builds 4.08× / 3.19× / 3.50× as fast, lowers median Python P95 by 76% / 5% / 78%, and delivers 2.37× / 0.74× / 2.61× Lance's batch throughput. Build time includes training, encoding, and serialization; Lance source-dataset writing is excluded. Lance uses its faster scheduling mode for each metric: partition parallelism 8 for P95 and 0 for batch QPS; recall ranges cover both modes.

-

Paimon's reader budget is 4 GiB and Lance's index-cache budget is 1 GiB; each measured index fits both. Both engines use native training defaults with the same sample count. GIST batch throughput remains about 26% below Lance, and its three-run P95 ranges overlap despite the 5% lower median. These results do not establish performance on cold storage, object stores, or other architectures and distributions. Pooled bounds can lose resolution on extreme-outlier data. See the full report and verification, raw measurements, and reproduction guide.

+

September 2026: optimization results

+

Three-run native ann_bench medians on Apple M4 Pro (12 CPU cores, 48 GiB RAM), with Rust 1.95 release builds, eight workers, nlist=1024, nprobe=64, k=10, 65,536 training rows, and 1,000 held-out public queries. Baseline commit 8dcabf2 and the current implementation use the same input files and settings; GloVe is L2-normalized.

+

Values show baseline → current. Build time includes training, encoding, and serialization. Sequential queries can reuse earlier partitions with the current 4 GiB reader budget; batch timing uses a separate fresh reader and includes payload reads and cache insertion. The baseline reader does not cache SQ partitions.

+
+ + + +
CorpusIndex build (ms)Query P95 (µs)Batch QPSRecall@10
SIFT1M934 → 886840 → 2986,417 → 8,9870.8626 → 0.9811
GIST1M6,404 → 5,8504,386 → 1,828899 → 9980.8576 → 0.9400
GloVe-100849 → 797739 → 2826,988 → 10,0090.8036 → 0.8760
+

File sizes are unchanged. These warm local-filesystem measurements do not establish performance on cold storage, object stores, or other architectures and distributions. Pooled bounds can lose resolution on extreme-outlier data. See the reproduction guide and the current build and local-search tables.

Historical uncached implementation

The following measurements predate pooled bounds and partition caching. They use nlist=1024, nprobe=64, k=10, and 12 Rayon workers on Apple M4 Pro. Keep them separate from the current eight-worker comparison above, which now includes a GIST rerun.

DatasetRecall@10Build / peak RSSWarm P95 / batch QPSRead/query
SIFT1M0.86273.93 s / 0.79 GiB0.79 ms / 11,0828.38 MiB
GIST1M0.857722.7 s / 5.09 GiB3.56 ms / 1,50270.95 MiB
GloVe-1000.80363.86 s / 0.71 GiB0.71 ms / 12,9626.99 MiB
diff --git a/docs/releases.html b/docs/releases.html index 531e309..75598c0 100644 --- a/docs/releases.html +++ b/docs/releases.html @@ -49,7 +49,7 @@

Distribution channels

Upcoming: 0.5.0

The repository is currently developing the 0.5.0 line. Until an ASF vote passes and the signed source archive appears under Apache downloads, code and packages from this line are development artifacts rather than an Apache release.

-

IVF-SQ now pools residual training bounds, fuses residual encoding, transposes output in bounded parallel batches, and reuses query heaps with conservative L2 block pruning. Unified readers and language bindings also cache decoded partitions within the existing reader memory budget. The SIFT1M/GIST1M/GloVe comparison with Lance 11.0.0 records build, Python query, and recall results with reproducible commands.

+

IVF-SQ now pools residual training bounds, fuses residual encoding, transposes output in bounded parallel batches, and reuses query heaps with conservative L2 block pruning. Unified readers and language bindings also cache decoded partitions within the existing reader memory budget. The SIFT1M/GIST1M/GloVe benchmarks record build, native query, and recall results with reproducible commands.

IVSQ v1 files remain compatible. Existing indexes receive reader optimizations without a rebuild; retrain and rebuild to use the new quantization bounds. Set the reader memory budget to zero to disable the SQ cache; direct Rust IVFSQIndexReader::open remains uncached. See reader options and upgrade details.

Rust IVF API migrationVersion 0.5.0 makes quantizer_centroids private on IVFFlatIndex, IVFPQIndex, IVFSQIndex, and IVFRQIndex. Replace direct reads with quantizer_centroids() and direct assignments with set_quantizer_centroids(...). The setter validates the centroid shape, rejects replacement after vectors are added, and refreshes cached derived state. IVF variants of VectorIndexConfig also require use_approximate_coarse_assignment; set it to true for the automatic 0.5.0 behavior or false for exact nearest-centroid assignment. Option-map callers can select the same policy with ivf.coarse-assignment=auto|exact. The policy is fixed when the writer is created; direct IVF indexes do not expose a post-training policy switch. These are source-level changes; the stored index format is unchanged.
diff --git a/tools/README.md b/tools/README.md index 78f85ac..17cac1c 100644 --- a/tools/README.md +++ b/tools/README.md @@ -53,19 +53,6 @@ neighbor ordering as cosine distance. Published neighbor IDs are copied unchanged. Conversion fails if a vector is zero-length or has a non-finite norm. -## Lance IVF-SQ comparison - -`benchmark_lance_ivfsq.py` benchmarks Lance 11+ against the same converted -`fvecs`/`ivecs` as `ann_bench`. It requires `pylance`, `numpy`, and `pyarrow`. -It creates a fresh dataset/index per repetition, warms selected partitions, -tests both default and partition-parallel query scheduling, and emits JSONL -with build time, file size, recall, P50/P95, and sequential/batch throughput. -`benchmark_ivfsq_reader.py` measures the matching public Paimon Python reader -and accepts a zero memory budget to isolate uncached performance. -Raw dataset writing is reported separately from index construction. See -[the IVF-SQ performance report](../docs/ivf-sq-performance.md) for commands, -measured results, and the differences between the public API entry points. - ## Java staging deploy `deploy_java_staging.sh` deploys the Java release candidate artifacts to Apache diff --git a/tools/benchmark_ivfsq_reader.py b/tools/benchmark_ivfsq_reader.py deleted file mode 100644 index 226f23b..0000000 --- a/tools/benchmark_ivfsq_reader.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Measure the public Python IVF-SQ reader, including positional-I/O callbacks. - -Build the native library in release mode, set PAIMON_VINDEX_LIB_PATH, and put -this checkout's python directory on PYTHONPATH. Uses the same fvecs/ivecs as -ann_bench and benchmark_lance_ivfsq.py. Emit one JSON row per repetition. -""" - -import argparse -import json -import os -from pathlib import Path -import time - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--index", type=Path, required=True) - parser.add_argument("--queries", type=Path, required=True) - parser.add_argument("--ground-truth", type=Path, required=True) - parser.add_argument("--nprobe", type=int, default=64) - parser.add_argument("--nq", type=int, default=1000) - parser.add_argument("--k", type=int, default=10) - parser.add_argument("--threads", type=int, default=8) - parser.add_argument("--repeats", type=int, default=3) - parser.add_argument("--memory-budget-bytes", type=int, default=4 * 1024**3) - args = parser.parse_args() - if min(args.nprobe, args.nq, args.k, args.threads, args.repeats) <= 0 or args.memory_budget_bytes < 0: - parser.error("counts must be positive; memory budget must be nonnegative") - os.environ["RAYON_NUM_THREADS"] = str(args.threads) - import numpy as np - from paimon_vindex import SearchParams, VectorIndexReader - - def read_vectors(path, dtype): - raw = np.memmap(path, mode="r", dtype=" args.nlist for p in args.nprobe): - parser.error("nprobe must be in 1..nlist") - if args.cache_bytes < 0 or any(p < 0 for p in args.query_parallelism): - parser.error("cache size and query parallelism must be nonnegative") - # Set pools before importing either native runtime. - for key in ["LANCE_CPU_THREADS", "LANCE_IO_THREADS", "RAYON_NUM_THREADS"]: - os.environ[key] = str(args.threads) - os.environ["OPENBLAS_NUM_THREADS"] = "1" - import lance - import numpy as np - import pyarrow as pa - - def read_vectors(path, dtype): - raw = np.memmap(path, mode="r", dtype=" len(base): - parser.error("inconsistent dimensions or train-n exceeds the base") - if not np.isfinite(base).all() or not np.isfinite(queries).all(): - parser.error("vectors must be finite") - if np.any(truth < 0) or np.any(truth >= len(base)): - parser.error("ground-truth IDs must refer to base rows") - table = pa.table({"vector": pa.FixedSizeListArray.from_arrays( - pa.array(base.ravel()), base.shape[1])}) - args.output_dir.mkdir(parents=True, exist_ok=True) - - def recall(results): - return sum(len(set(r) & set(g)) for r, g in zip(results, truth)) / truth.size - - for repeat in range(args.repeats): - path = Path(tempfile.mkdtemp(prefix="lance-ivfsq-", dir=args.output_dir)) / "data.lance" - started = time.perf_counter() - dataset = lance.write_dataset(table, str(path)) - data_write_s = time.perf_counter() - started - started = time.perf_counter() - dataset.create_index("vector", "IVF_SQ", metric="L2", - num_partitions=args.nlist, - sample_rate=args.train_n // args.nlist) - index_build_s = time.perf_counter() - started - dataset = lance.dataset(str(path), index_cache_size_bytes=args.cache_bytes) - index_bytes = sum(p.stat().st_size for p in (path / "_indices").rglob("*") if p.is_file()) - for nprobe in args.nprobe: - for parallelism in args.query_parallelism: - def search(q): - return dataset.to_table(columns=["_distance"], with_row_id=True, - nearest={"column": "vector", "q": q, - "k": args.k, "nprobes": nprobe, - "query_parallelism": parallelism}) - # Warm the union of selected partitions. Leave refine_factor unset: - # returning IDs/distances must not fetch or rerank raw vectors. - search(queries) - latencies, results = [], [] - for query in queries: - started = time.perf_counter() - result = search(query) - latencies.append(time.perf_counter() - started) - results.append(result["_rowid"].to_numpy()) - batch_s = [] - for _ in range(3): - started = time.perf_counter() - batch = search(queries) - batch_s.append(time.perf_counter() - started) - query_index = batch["query_index"].to_numpy() - row_ids = batch["_rowid"].to_numpy() - batch_results = [row_ids[query_index == i] for i in range(args.nq)] - print(json.dumps({ - "engine": "lance", "version": lance.__version__, - "machine": platform.machine(), "repeat": repeat, - "base": str(args.base), "n": len(base), "d": base.shape[1], - "nq": args.nq, "k": args.k, "nlist": args.nlist, - "train_n": args.train_n, "threads": args.threads, - "nprobe": nprobe, "query_parallelism": parallelism, - "data_write_s": data_write_s, "index_build_s": index_build_s, - "index_bytes": index_bytes, "recall": recall(results), - "p50_ms": float(np.percentile(latencies, 50) * 1000), - "p95_ms": float(np.percentile(latencies, 95) * 1000), - "sequential_qps": args.nq / sum(latencies), - "batch_s": batch_s, "batch_qps": args.nq / float(np.median(batch_s)), - "batch_recall": recall(batch_results), "index_path": str(path), - }), flush=True) - - -if __name__ == "__main__": - main() From 6603e4e73edc51575f7859e73492f90613e1e3c0 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Sun, 6 Sep 2026 20:23:05 +0800 Subject: [PATCH 3/3] refactor: initialize IVF-SQ cache during reader construction --- core/src/index.rs | 8 ++- core/src/ivfsq_io.rs | 119 +++++++++++++++++++++++++++++-------------- 2 files changed, 83 insertions(+), 44 deletions(-) diff --git a/core/src/index.rs b/core/src/index.rs index 5d3abca..773984f 100644 --- a/core/src/index.rs +++ b/core/src/index.rs @@ -1464,11 +1464,9 @@ impl VectorIndexReader { IVFFLAT_MAGIC => Ok(Self::IvfFlat(IVFFlatIndexReader::open_with_header( reader, header, )?)), - IVF_SQ_MAGIC => { - let mut reader = IVFSQIndexReader::open_with_header(reader, header)?; - reader.configure_cache(options.memory_budget_bytes); - Ok(Self::IvfSq(reader)) - } + IVF_SQ_MAGIC => Ok(Self::IvfSq(IVFSQIndexReader::open_with_header_and_options( + reader, header, options, + )?)), MAGIC => Ok(Self::IvfPq(IVFPQIndexReader::open_with_header( reader, header, )?)), diff --git a/core/src/ivfsq_io.rs b/core/src/ivfsq_io.rs index 5607a67..5daf168 100644 --- a/core/src/ivfsq_io.rs +++ b/core/src/ivfsq_io.rs @@ -182,38 +182,22 @@ pub struct IVFSQIndexReader { } impl IVFSQIndexReader { - /// Open with a bounded cache of decoded partitions. `open` retains the - /// uncached positional-I/O behavior for callers that manage their own cache. - pub fn open_with_options(reader: R, options: VectorIndexReaderOptions) -> io::Result { - let mut index = Self::open(reader)?; - index.configure_cache(options.memory_budget_bytes); - Ok(index) - } - - pub(crate) fn configure_cache(&mut self, memory_budget_bytes: usize) { - let resident = size_of::() - + self.quantizer_centroids.capacity() * size_of::() - + self.list_offsets.capacity() * size_of::() - + self.list_counts.capacity() * size_of::() - + self.list_id_bytes_lens.capacity() * size_of::() - + self.list_sqs.capacity() * size_of::() - + std::iter::once(&self.sq) - .chain(&self.list_sqs) - .map(|sq| (sq.mins.capacity() + sq.maxs.capacity()) * size_of::()) - .sum::(); - self.list_cache = - SqListCache::new(self.nlist, memory_budget_bytes.saturating_sub(resident)); + pub fn open(reader: R) -> io::Result { + Self::open_with_options(reader, VectorIndexReaderOptions::new(0)) } - pub fn open(mut reader: R) -> io::Result { + /// Open with a bounded cache of decoded partitions. `open` retains the + /// uncached positional-I/O behavior for callers that manage their own cache. + pub fn open_with_options(mut reader: R, options: VectorIndexReaderOptions) -> io::Result { let mut header = [0u8; IVF_SQ_HEADER_SIZE]; reader.pread(&mut [ReadRequest::new(0, &mut header)])?; - Self::open_with_header(reader, header) + Self::open_with_header_and_options(reader, header, options) } - pub(crate) fn open_with_header( + pub(crate) fn open_with_header_and_options( mut reader: R, header: [u8; IVF_SQ_HEADER_SIZE], + options: VectorIndexReaderOptions, ) -> io::Result { let read_u32 = |offset: usize| u32::from_le_bytes(header[offset..offset + 4].try_into().unwrap()); @@ -389,6 +373,19 @@ impl IVFSQIndexReader { )); } + let resident = size_of::() + + quantizer_centroids.capacity() * size_of::() + + list_offsets.capacity() * size_of::() + + list_counts.capacity() * size_of::() + + list_id_bytes_lens.capacity() * size_of::() + + list_sqs.capacity() * size_of::() + + std::iter::once(&sq) + .chain(&list_sqs) + .map(|sq| (sq.mins.capacity() + sq.maxs.capacity()) * size_of::()) + .sum::(); + let list_cache = + SqListCache::new(nlist, options.memory_budget_bytes.saturating_sub(resident)); + Ok(Self { reader, d, @@ -402,7 +399,7 @@ impl IVFSQIndexReader { list_counts, list_id_bytes_lens, loaded: true, - list_cache: None, + list_cache, }) } @@ -1633,20 +1630,64 @@ mod tests { } #[test] - fn ivfsq_open_coalesces_resident_metadata() { - let (index, _, _) = build_index(8, 32, 512); - let calls = Arc::new(AtomicUsize::new(0)); - let source = CountingReader { - inner: Cursor::new(serialized_index(&index)), - calls: Arc::clone(&calls), - }; - let mut reader = IVFSQIndexReader::open(source).unwrap(); - reader.optimize_for_search().unwrap(); - assert_eq!( - calls.load(Ordering::SeqCst), - 2, - "direct IVF-SQ open should use one header read and one resident-metadata read" - ); + fn ivfsq_reader_entry_points_preserve_cache_policy_and_header_reads() { + use crate::index::VectorIndexReader; + + let (index, data, _) = build_index(8, 8, 512); + let bytes = serialized_index(&index); + let query = &data[..8]; + let expected = IVFSQIndexReader::open(Cursor::new(bytes.clone())) + .unwrap() + .search(query, 5, 8) + .unwrap(); + for (unified, budget, cached) in [ + (false, None, false), + (false, Some(0), false), + (false, Some(1), false), + (false, Some(1024 * 1024), true), + (true, None, true), + (true, Some(0), false), + (true, Some(1), false), + (true, Some(1024 * 1024), true), + ] { + let calls = Arc::new(AtomicUsize::new(0)); + let source = CountingReader { + inner: Cursor::new(bytes.clone()), + calls: Arc::clone(&calls), + }; + let options = budget.map(VectorIndexReaderOptions::new); + let mut reader = if unified { + let reader = match options { + Some(options) => VectorIndexReader::open_with_options(source, options), + None => VectorIndexReader::open(source), + } + .unwrap(); + let VectorIndexReader::IvfSq(reader) = reader else { + panic!("SQ file must dispatch to the SQ reader"); + }; + reader + } else { + match options { + Some(options) => IVFSQIndexReader::open_with_options(source, options), + None => IVFSQIndexReader::open(source), + } + .unwrap() + }; + reader.optimize_for_search().unwrap(); + assert_eq!( + calls.swap(0, Ordering::Relaxed), + 2, + "open should read the header and resident metadata exactly once" + ); + assert_eq!(reader.search(query, 5, 8).unwrap(), expected); + assert_eq!(calls.swap(0, Ordering::Relaxed), 1); + assert_eq!(reader.search(query, 5, 8).unwrap(), expected); + assert_eq!( + calls.load(Ordering::Relaxed), + usize::from(!cached), + "cache policy for unified={unified}, budget={budget:?}" + ); + } } #[test]