From 7c1e82b88cd1f9dd70d3d5a6b1e2d8c25c922bbf Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 9 Sep 2026 19:29:54 +0800 Subject: [PATCH] [chore](build) Sync Lance-C thirdparty patches from branch-41 ### What problem does this PR solve? Problem Summary: The current branch bundles only the older Lance-C v0.1.9 PR #73/#74 patches. Port the Lance-C thirdparty state from branch-41 lance_memory at d9fc77680774baf3e9283f8dcd6c1f78490de8b6, including scanner options, the Lance v11 upgrade, refreshed data-cache support, and scalar-index segment APIs. Apply PR #74, #75/#78, #77, #73, then #79 so dependent patches apply cleanly without changing their source bytes. Only thirdparty changes are included from the mixed source commits. Source commits: - a425660dfebae4b1d928b14c1d1f4a68ead08eb2 - e078d4f9729f71f9b60fb8826ac59a6803e9d50b - d9fc77680774baf3e9283f8dcd6c1f78490de8b6 ### Release note Update bundled Lance internals to v11 and include the latest Lance-C cache and scanner APIs from the source branch. ### Check List (For Author) - Test: Manual validation of archive MD5, strict zero-fuzz application of the full patch chain, isolated downloader execution and repeat execution, resulting source-tree equality, source patch byte equality, and bash -n. Downloader whitespace checks passed; outer diff whitespace warnings in imported patch payloads are retained to preserve exact source bytes. No Doris or Lance-C builds or unit/regression tests were run. - Behavior changed: Yes; update the bundled dependency and patch order. - Does this need documentation: No; dependency-only propagation. --- thirdparty/download-thirdparty.sh | 20 +- thirdparty/patches/lance-c-0.1.9-pr-73.patch | 133 +- .../patches/lance-c-0.1.9-pr-75-pr-78.patch | 2441 +++++++++++++++++ thirdparty/patches/lance-c-0.1.9-pr-77.patch | 1863 +++++++++++++ thirdparty/patches/lance-c-0.1.9-pr-79.patch | 1782 ++++++++++++ 5 files changed, 6177 insertions(+), 62 deletions(-) create mode 100644 thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch create mode 100644 thirdparty/patches/lance-c-0.1.9-pr-77.patch create mode 100644 thirdparty/patches/lance-c-0.1.9-pr-79.patch diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index eb4aed22823e63..9d45bfd25035fe 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -763,19 +763,19 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# Apply Doris lance-c patches. +# Apply Doris lance-c patches as one chain to the pinned release archive. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then - if [[ "${LANCE_C_SOURCE}" == "lance-c-0.1.9" ]]; then - cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" - if [[ ! -f "${PATCHED_MARK}" ]]; then - patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ - -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-73.patch" + cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + # Apply the merged PRs first; the latest PR #73 and #79 both require Lance v11. + # This order keeps the upstream patches unchanged, including Cargo.lock. + for lance_patch in pr-74 pr-75-pr-78 pr-77 pr-73 pr-79; do patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ - -p1 <"${TP_PATCH_DIR}/lance-c-0.1.9-pr-74.patch" - touch "${PATCHED_MARK}" - fi - cd - + -p1 <"${TP_PATCH_DIR}/${LANCE_C_SOURCE}-${lance_patch}.patch" + done + touch "${PATCHED_MARK}" fi + cd - echo "Finished patching ${LANCE_C_SOURCE}" fi diff --git a/thirdparty/patches/lance-c-0.1.9-pr-73.patch b/thirdparty/patches/lance-c-0.1.9-pr-73.patch index 7d5eb45ad6ce7d..33a8985b25aecc 100644 --- a/thirdparty/patches/lance-c-0.1.9-pr-73.patch +++ b/thirdparty/patches/lance-c-0.1.9-pr-73.patch @@ -1,10 +1,10 @@ -From a4c71309ddb76ad79808e3e8f7797bcd9bfc174a Mon Sep 17 00:00:00 2001 +From cc373477a6485cb24ed7ea88ef506e93caacedd0 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 3 Sep 2026 13:00:43 +0800 Subject: [PATCH] foyer --- - Cargo.lock | 205 ++++++ + Cargo.lock | 207 +++++- Cargo.toml | 4 + README.md | 22 + include/lance/lance.h | 63 ++ @@ -19,32 +19,28 @@ Subject: [PATCH] foyer tests/c_api_test.rs | 221 +++++++ tests/cpp/test_c_api.c | 32 + tests/cpp/test_cpp_api.cpp | 23 + - 15 files changed, 1918 insertions(+), 1 deletion(-) + 15 files changed, 1917 insertions(+), 4 deletions(-) create mode 100644 src/data_cache.rs create mode 100644 src/foyer_data_cache.rs diff --git a/Cargo.lock b/Cargo.lock -index 60c1caf..85536f5 100644 +index bc37cb9..199f997 100644 --- a/Cargo.lock +++ b/Cargo.lock -@@ -434,6 +434,16 @@ dependencies = [ - "loom", +@@ -444,6 +444,12 @@ dependencies = [ + "slab", ] +[[package]] +name = "asyncband" -+version = "0.6.7" ++version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" -+dependencies = [ -+ "hashbrown 0.17.1", -+ "slab", -+] ++checksum = "2e52766975a4f080528a898235c51e82e65df9db713419067b5368040eeb5659" + [[package]] name = "atoi" version = "2.0.0" -@@ -1267,6 +1277,17 @@ version = "0.8.7" +@@ -1293,6 +1299,17 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" @@ -62,7 +58,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "countio" version = "0.3.0" -@@ -2112,6 +2133,12 @@ dependencies = [ +@@ -2148,6 +2165,12 @@ dependencies = [ "url", ] @@ -73,9 +69,9 @@ index 60c1caf..85536f5 100644 +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] - name = "der" - version = "0.7.10" -@@ -2298,6 +2325,16 @@ version = "0.2.3" + name = "defmt" + version = "1.1.1" +@@ -2366,6 +2389,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" @@ -92,18 +88,18 @@ index 60c1caf..85536f5 100644 [[package]] name = "fastrand" version = "2.3.0" -@@ -2369,6 +2406,127 @@ dependencies = [ +@@ -2437,12 +2470,133 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "cab5c4bac30455a0dbc4c858436fb440d51bc6543daafbdf35c5e6ca94c0afd7" ++checksum = "f911e6f0b4909f23d65a95c5d27bcf2f92855b8a0f629b47b743d53d14f2828b" +dependencies = [ + "anyhow", -+ "asyncband", ++ "asyncband 0.7.1", + "equivalent", + "foyer-common", + "foyer-memory", @@ -118,9 +114,9 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-common" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "e3634c6f3da978b6cae98d54367a2342041d3a4786b20c0384164cc7fce94787" ++checksum = "05cdcae6cedec72c28e97ada0b453e33b1df57fbc209708091107be2a283d577" +dependencies = [ + "anyhow", + "bytes", @@ -143,13 +139,13 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-memory" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "5349a61af676b3275bfef7a5fceceeadf6b0a7dd9ec14ba0edd78e1ff771d101" ++checksum = "8a51c8ce8e1e323a1e087ac45bf629d953676fc5e0037d14a799fadd272b42db" +dependencies = [ + "anyhow", -+ "asyncband", -+ "bitflags", ++ "asyncband 0.7.1", ++ "bitflags 2.11.0", + "datasketches", + "equivalent", + "foyer-common", @@ -168,13 +164,13 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-storage" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "49cfcce3c10a1f2ac65bfcf0bd85501462d454c594c8a16a4a27cce773599381" ++checksum = "127a64057f63e361123cf62b3fd50a36147783687d4cd36e7087c27f9909265b" +dependencies = [ + "allocator-api2", + "anyhow", -+ "asyncband", ++ "asyncband 0.7.1", + "bytes", + "core_affinity", + "equivalent", @@ -200,13 +196,19 @@ index 60c1caf..85536f5 100644 + +[[package]] +name = "foyer-tokio" -+version = "0.22.4" ++version = "0.22.5" +source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "6b7315103199f3415a010befd6dd5a1c8e7082fbc49c0194931e65629995d9d1" ++checksum = "cbdbb9f39443cb348a069baa1a0ec73bcea848a4a383eb2da1a5ea7a0ca05941" +dependencies = [ + "tokio", +] + + [[package]] + name = "frostem" + version = "1.20260821.5" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f" + +[[package]] +name = "fs4" +version = "0.13.1" @@ -220,7 +222,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "fs_extra" version = "1.3.0" -@@ -3392,6 +3550,15 @@ dependencies = [ +@@ -3485,6 +3639,15 @@ dependencies = [ "either", ] @@ -236,7 +238,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "itoa" version = "1.0.18" -@@ -3703,8 +3870,11 @@ dependencies = [ +@@ -3788,8 +3951,11 @@ dependencies = [ "arrow", "arrow-array", "arrow-schema", @@ -248,7 +250,7 @@ index 60c1caf..85536f5 100644 "futures", "half", "lance", -@@ -3718,6 +3888,7 @@ dependencies = [ +@@ -3803,6 +3969,7 @@ dependencies = [ "lance-table", "libc", "log", @@ -256,9 +258,9 @@ index 60c1caf..85536f5 100644 "pin-project", "prost", "snafu", -@@ -4399,6 +4570,15 @@ version = "2.8.0" - source = "registry+https://github.com/rust-lang/crates.io-index" - checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +@@ -4492,6 +4659,15 @@ dependencies = [ + "libc", + ] +[[package]] +name = "memoffset" @@ -272,7 +274,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "mime" version = "0.3.17" -@@ -4436,6 +4616,16 @@ dependencies = [ +@@ -4529,6 +4705,16 @@ dependencies = [ "windows-sys 0.61.2", ] @@ -289,7 +291,34 @@ index 60c1caf..85536f5 100644 [[package]] name = "moka" version = "0.12.15" -@@ -6547,6 +6737,12 @@ version = "0.4.12" +@@ -4840,7 +5026,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" + dependencies = [ + "anyhow", +- "asyncband", ++ "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", +@@ -4879,7 +5065,7 @@ version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" + dependencies = [ +- "asyncband", ++ "asyncband 0.6.7", + "futures", + "http 1.4.0", + "opendal-core", +@@ -4943,7 +5129,7 @@ version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" + dependencies = [ +- "asyncband", ++ "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "http 1.4.0", +@@ -6668,6 +6854,12 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" @@ -302,7 +331,7 @@ index 60c1caf..85536f5 100644 [[package]] name = "smallvec" version = "1.15.1" -@@ -7831,6 +8027,15 @@ dependencies = [ +@@ -7930,6 +8122,15 @@ dependencies = [ "windows-targets 0.52.6", ] @@ -319,7 +348,7 @@ index 60c1caf..85536f5 100644 name = "windows-sys" version = "0.60.2" diff --git a/Cargo.toml b/Cargo.toml -index d072a5d..342928c 100644 +index 3920a65..356c9dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,8 @@ datafusion = { version = "54.0.0", default-features = false } @@ -335,7 +364,7 @@ index d072a5d..342928c 100644 half = "2" tokio = { version = "1", features = ["rt-multi-thread", "sync"] } futures = "0.3" -+foyer = "=0.22.4" ++foyer = "=0.22.5" log = "0.4" libc = "0.2" +object_store = "0.13.2" @@ -383,7 +412,7 @@ index 2056671..d9a5f2b 100644 `lance_dataset_open` takes a `version` argument — `0` means the latest, any diff --git a/include/lance/lance.h b/include/lance/lance.h -index 3bf291f..e4c593e 100644 +index 31213da..152351b 100644 --- a/include/lance/lance.h +++ b/include/lance/lance.h @@ -206,6 +206,36 @@ typedef struct LanceSessionCacheStats { @@ -471,10 +500,10 @@ index 3bf291f..e4c593e 100644 void lance_dataset_close(LanceDataset* dataset); diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp -index 6cf245f..c1102e4 100644 +index 404d2df..3fe9dbc 100644 --- a/include/lance/lance.hpp +++ b/include/lance/lance.hpp -@@ -171,6 +171,13 @@ struct SqlColumn { +@@ -176,6 +176,13 @@ struct SqlColumn { // ─── Shared Session ────────────────────────────────────────────────────────── @@ -488,7 +517,7 @@ index 6cf245f..c1102e4 100644 class Session { Handle handle_; -@@ -180,6 +187,21 @@ class Session { +@@ -185,6 +192,21 @@ class Session { if (!handle_) check_error(); } @@ -510,7 +539,7 @@ index 6cf245f..c1102e4 100644 LanceSessionCacheStats cache_stats() const { LanceSessionCacheStats stats{}; if (lance_session_get_cache_stats(handle_.get(), &stats) != 0) -@@ -260,6 +282,13 @@ class Dataset { +@@ -265,6 +287,13 @@ class Dataset { return Dataset(ds); } @@ -1975,7 +2004,7 @@ index 1971510..ba51c87 100644 // SAFETY: `out_dataset` is non-NULL (checked above) and the caller // guarantees it points to caller-owned, writable storage of size diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs -index 8805764..b4313f4 100644 +index bde742d..a4ea8f8 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs @@ -96,10 +96,83 @@ fn create_large_dataset(num_rows: i32) -> (tempfile::TempDir, String) { @@ -2183,7 +2212,7 @@ index 8805764..b4313f4 100644 #[test] fn test_open_nonexistent() { let c_uri = c_str("memory://nonexistent_dataset_xyz"); -@@ -2787,6 +2974,40 @@ fn test_dataset_restore_to_prior_version() { +@@ -2977,6 +3164,40 @@ fn test_dataset_restore_to_prior_version() { unsafe { lance_dataset_close(ds) }; } @@ -2275,7 +2304,7 @@ index c49ecfa..dd674eb 100644 test_scan_with_limit(uri); test_versions(uri); diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp -index 17b1ab6..e1aadbd 100644 +index 28762b8..5f34bde 100644 --- a/tests/cpp/test_cpp_api.cpp +++ b/tests/cpp/test_cpp_api.cpp @@ -99,6 +99,28 @@ static void test_shared_session(const std::string& uri) { @@ -2307,7 +2336,7 @@ index 17b1ab6..e1aadbd 100644 static void test_dataset_schema(const std::string& uri) { TEST(test_dataset_schema); -@@ -922,6 +944,7 @@ int main(int argc, char** argv) { +@@ -929,6 +951,7 @@ int main(int argc, char** argv) { test_dataset_open(uri); test_shared_session(uri); diff --git a/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch b/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch new file mode 100644 index 00000000000000..08687627e7ea21 --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-75-pr-78.patch @@ -0,0 +1,2441 @@ +Lance-C v0.1.9 scanner options: PR #75 followed by PR #78. +Apply after lance-c-0.1.9-pr-74.patch, before lance-c-0.1.9-pr-77.patch. +The upstream mail patches below are concatenated without modification. + +PR #75: https://github.com/lance-format/lance-c/pull/75 +Head: 043a1f7eac253d8ac6be3f970b60fcbf615295aa +PR #78: https://github.com/lance-format/lance-c/pull/78 +Head: e894f591aef358cd36fdbf915c4d5b95fd0e8348 + +From 0752592cc4bbc69f8f9333362a00f3bceee73100 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Fri, 4 Sep 2026 23:35:29 +0800 +Subject: [PATCH 1/2] add some setter + +--- + include/lance/lance.h | 81 +++++++++++ + include/lance/lance.hpp | 47 +++++++ + src/scanner.rs | 279 +++++++++++++++++++++++++++++++++++++ + tests/c_api_test.rs | 230 ++++++++++++++++++++++++++++++ + tests/cpp/test_cpp_api.cpp | 7 + + 5 files changed, 644 insertions(+) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 3bf291f..00f415b 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -934,6 +934,74 @@ LanceScanner* lance_scanner_new( + int32_t lance_scanner_set_limit(LanceScanner* scanner, int64_t limit); + int32_t lance_scanner_set_offset(LanceScanner* scanner, int64_t offset); + int32_t lance_scanner_set_batch_size(LanceScanner* scanner, int64_t batch_size); ++ ++/** ++ * Set the target output batch size in bytes. ++ * ++ * When set, this takes precedence over the row-based batch size. The value ++ * must be greater than zero and must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_batch_size_bytes( ++ LanceScanner* scanner, ++ uint64_t batch_size_bytes ++); ++ ++/** ++ * Set the scanner I/O buffer size in bytes. ++ * ++ * The value must be greater than zero and must be set before scanning starts. ++ * This bounds buffered I/O received from storage, but is not a hard limit on ++ * all memory used by the scanner. ++ * ++ * @param scanner Scanner handle. Must not be NULL. ++ * @param io_buffer_size_bytes I/O buffer size in bytes. Must be greater than zero. ++ * @return 0 on success, -1 on error. ++ */ ++int32_t lance_scanner_set_io_buffer_size( ++ LanceScanner* scanner, ++ uint64_t io_buffer_size_bytes ++); ++ ++/** ++ * Set the maximum number of batches decoded concurrently. ++ * ++ * @param batch_readahead Number of in-flight batch decode tasks. Must be greater than zero. ++ */ ++int32_t lance_scanner_set_batch_readahead( ++ LanceScanner* scanner, ++ size_t batch_readahead ++); ++ ++/** ++ * Set fragment readahead for unordered scans. ++ * ++ * This setting is only used when scan-in-order is disabled. The value must be ++ * greater than zero. ++ */ ++int32_t lance_scanner_set_fragment_readahead( ++ LanceScanner* scanner, ++ size_t fragment_readahead ++); ++ ++/** ++ * Set the target number of physical execution partitions. ++ * ++ * This controls the partition count used by the physical optimizer and can be ++ * used to bound scan CPU parallelism. The value must be greater than zero and ++ * must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_target_parallelism( ++ LanceScanner* scanner, ++ size_t target_parallelism ++); ++ ++/** ++ * Configure whether batches are returned in storage order (default: true). ++ * ++ * Disabling ordering can improve throughput by returning batches as soon as ++ * they are ready. ++ */ ++int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_order); + int32_t lance_scanner_with_row_id(LanceScanner* scanner, bool enable); + + /** +@@ -1656,6 +1724,19 @@ int32_t lance_scanner_nearest( + ); + + int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n); ++ ++/** ++ * Set vector index partition-search concurrency for each query. ++ * ++ * A value of -1 uses the CPU pool size, 0 selects Lance's automatic policy, ++ * 1 uses the sequential path, and values greater than 1 request parallel ++ * partition search. The effective value is capped by available parallelism. ++ * Values below -1 are rejected. Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_query_parallelism( ++ LanceScanner* scanner, ++ int32_t query_parallelism ++); + int32_t lance_scanner_set_refine_factor(LanceScanner* scanner, uint32_t f); + int32_t lance_scanner_set_ef(LanceScanner* scanner, uint32_t e); + int32_t lance_scanner_set_metric(LanceScanner* scanner, LanceMetricType metric); +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 6cf245f..3c03f86 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1196,6 +1196,48 @@ class Scanner { + return *this; + } + ++ /// Set the target output batch size in bytes. ++ Scanner& batch_size_bytes(uint64_t bytes) { ++ if (lance_scanner_set_batch_size_bytes(handle_.get(), bytes) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set the scanner I/O buffer size in bytes. ++ Scanner& io_buffer_size(uint64_t bytes) { ++ if (lance_scanner_set_io_buffer_size(handle_.get(), bytes) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set the number of batches decoded concurrently. ++ Scanner& batch_readahead(size_t batches) { ++ if (lance_scanner_set_batch_readahead(handle_.get(), batches) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set fragment readahead for unordered scans. ++ Scanner& fragment_readahead(size_t fragments) { ++ if (lance_scanner_set_fragment_readahead(handle_.get(), fragments) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Set the target number of physical execution partitions. ++ Scanner& target_parallelism(size_t partitions) { ++ if (lance_scanner_set_target_parallelism(handle_.get(), partitions) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether batches are returned in storage order. ++ Scanner& scan_in_order(bool ordered = true) { ++ if (lance_scanner_set_scan_in_order(handle_.get(), ordered) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Enable/disable row ID in output. + Scanner& with_row_id(bool enable = true) { + if (lance_scanner_with_row_id(handle_.get(), enable) != 0) +@@ -1313,6 +1355,11 @@ class Scanner { + if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error(); + return *this; + } ++ Scanner& query_parallelism(int32_t parallelism) { ++ if (lance_scanner_set_query_parallelism(handle_.get(), parallelism) != 0) ++ check_error(); ++ return *this; ++ } + Scanner& refine_factor(uint32_t f) { + if (lance_scanner_set_refine_factor(handle_.get(), f) != 0) check_error(); + return *this; +diff --git a/src/scanner.rs b/src/scanner.rs +index 0c29b17..ebedafc 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -60,11 +60,18 @@ pub struct LanceScanner { + limit: Option, + offset: Option, + batch_size: Option, ++ batch_size_bytes: Option, ++ io_buffer_size: Option, ++ batch_readahead: Option, ++ fragment_readahead: Option, ++ target_parallelism: Option, ++ scan_in_order: Option, + with_row_id: bool, + fragment_ids: Option>, + index_segments: Option>, + nearest: Option, + nprobes: Option, ++ query_parallelism: Option, + refine_factor: Option, + ef: Option, + metric_override: Option, +@@ -129,11 +136,18 @@ impl LanceScanner { + limit: None, + offset: None, + batch_size: None, ++ batch_size_bytes: None, ++ io_buffer_size: None, ++ batch_readahead: None, ++ fragment_readahead: None, ++ target_parallelism: None, ++ scan_in_order: None, + with_row_id: false, + fragment_ids: None, + index_segments: None, + nearest: None, + nprobes: None, ++ query_parallelism: None, + refine_factor: None, + ef: None, + metric_override: None, +@@ -164,6 +178,15 @@ impl LanceScanner { + Arc::clone(&self.poisoned) + } + ++ fn ensure_scan_not_started(&self, setting_name: &str) -> Result<()> { ++ if self.scan_started.load(Ordering::Acquire) { ++ return Err(lance_core::Error::invalid_input_source( ++ format!("{setting_name} must be set before the scan starts").into(), ++ )); ++ } ++ Ok(()) ++ } ++ + /// Apply fragment selection to a scanner builder if fragment_ids is set. + fn apply_fragment_filter(&self, scanner: &mut lance::dataset::scanner::Scanner) -> Result<()> { + if let Some(ids) = &self.fragment_ids { +@@ -231,6 +254,24 @@ impl LanceScanner { + if let Some(bs) = self.batch_size { + scanner.batch_size(bs); + } ++ if let Some(batch_size_bytes) = self.batch_size_bytes { ++ scanner.batch_size_bytes(batch_size_bytes); ++ } ++ if let Some(io_buffer_size) = self.io_buffer_size { ++ scanner.io_buffer_size(io_buffer_size); ++ } ++ if let Some(batch_readahead) = self.batch_readahead { ++ scanner.batch_readahead(batch_readahead); ++ } ++ if let Some(fragment_readahead) = self.fragment_readahead { ++ scanner.fragment_readahead(fragment_readahead); ++ } ++ if let Some(target_parallelism) = self.target_parallelism { ++ scanner.target_parallelism(target_parallelism); ++ } ++ if let Some(scan_in_order) = self.scan_in_order { ++ scanner.scan_in_order(scan_in_order); ++ } + if self.with_row_id { + scanner.with_row_id(); + } +@@ -260,6 +301,9 @@ impl LanceScanner { + if let Some(np) = self.nprobes { + scanner.nprobes(np as usize); + } ++ if let Some(query_parallelism) = self.query_parallelism { ++ scanner.query_parallelism(query_parallelism); ++ } + if let Some(rf) = self.refine_factor { + scanner.refine(rf); + } +@@ -757,6 +801,205 @@ unsafe fn scanner_set_batch_size_inner(scanner: *mut LanceScanner, batch_size: i + Ok(0) + } + ++/// Set the target output batch size in bytes. Returns 0 on success. ++/// ++/// The size must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_batch_size_bytes( ++ scanner: *mut LanceScanner, ++ batch_size_bytes: u64, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_batch_size_bytes_inner(scanner, batch_size_bytes) ++ }) ++} ++ ++unsafe fn scanner_set_batch_size_bytes_inner( ++ scanner: *mut LanceScanner, ++ batch_size_bytes: u64, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if batch_size_bytes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "batch_size_bytes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("batch_size_bytes")?; ++ scanner.batch_size_bytes = Some(batch_size_bytes); ++ Ok(0) ++} ++ ++/// Set the scanner I/O buffer size in bytes. Returns 0 on success. ++/// ++/// The size must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_io_buffer_size( ++ scanner: *mut LanceScanner, ++ io_buffer_size_bytes: u64, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_io_buffer_size_inner(scanner, io_buffer_size_bytes) ++ }) ++} ++ ++unsafe fn scanner_set_io_buffer_size_inner( ++ scanner: *mut LanceScanner, ++ io_buffer_size_bytes: u64, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if io_buffer_size_bytes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "io_buffer_size_bytes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("io_buffer_size_bytes")?; ++ scanner.io_buffer_size = Some(io_buffer_size_bytes); ++ Ok(0) ++} ++ ++/// Set the number of batches to decode concurrently. Returns 0 on success. ++/// ++/// The value must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_batch_readahead( ++ scanner: *mut LanceScanner, ++ batch_readahead: usize, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_batch_readahead_inner(scanner, batch_readahead) ++ }) ++} ++ ++unsafe fn scanner_set_batch_readahead_inner( ++ scanner: *mut LanceScanner, ++ batch_readahead: usize, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if batch_readahead == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "batch_readahead must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("batch_readahead")?; ++ scanner.batch_readahead = Some(batch_readahead); ++ Ok(0) ++} ++ ++/// Set the number of fragments to read ahead for unordered scans. ++/// ++/// The value must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_fragment_readahead( ++ scanner: *mut LanceScanner, ++ fragment_readahead: usize, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_fragment_readahead_inner(scanner, fragment_readahead) ++ }) ++} ++ ++unsafe fn scanner_set_fragment_readahead_inner( ++ scanner: *mut LanceScanner, ++ fragment_readahead: usize, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if fragment_readahead == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "fragment_readahead must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("fragment_readahead")?; ++ scanner.fragment_readahead = Some(fragment_readahead); ++ Ok(0) ++} ++ ++/// Set the target number of physical execution partitions. ++/// ++/// The value must be greater than zero and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_target_parallelism( ++ scanner: *mut LanceScanner, ++ target_parallelism: usize, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_target_parallelism_inner(scanner, target_parallelism) ++ }) ++} ++ ++unsafe fn scanner_set_target_parallelism_inner( ++ scanner: *mut LanceScanner, ++ target_parallelism: usize, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if target_parallelism == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "target_parallelism must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("target_parallelism")?; ++ scanner.target_parallelism = Some(target_parallelism); ++ Ok(0) ++} ++ ++/// Configure whether scan results are returned in storage order. ++/// ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_scan_in_order( ++ scanner: *mut LanceScanner, ++ scan_in_order: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_scan_in_order_inner(scanner, scan_in_order) ++ }) ++} ++ ++unsafe fn scanner_set_scan_in_order_inner( ++ scanner: *mut LanceScanner, ++ scan_in_order: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("scan_in_order")?; ++ scanner.scan_in_order = Some(scan_in_order); ++ Ok(0) ++} ++ + /// Enable or disable row ID in scan output. Returns 0. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_with_row_id( +@@ -1766,6 +2009,42 @@ scanner_set_u32!(lance_scanner_set_nprobes, nprobes); + scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor); + scanner_set_u32!(lance_scanner_set_ef, ef); + ++/// Set vector index partition-search concurrency for each query. ++/// ++/// `-1` uses the CPU pool size, `0` selects Lance's automatic policy, and ++/// positive values request that many workers. Values below `-1` are invalid. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_query_parallelism( ++ scanner: *mut LanceScanner, ++ query_parallelism: i32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_query_parallelism_inner(scanner, query_parallelism) ++ }) ++} ++ ++unsafe fn scanner_set_query_parallelism_inner( ++ scanner: *mut LanceScanner, ++ query_parallelism: i32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if query_parallelism < -1 { ++ return Err(lance_core::Error::invalid_input_source( ++ format!("query_parallelism must be -1, 0, or greater than 0, got {query_parallelism}") ++ .into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("query_parallelism")?; ++ scanner.query_parallelism = Some(query_parallelism); ++ Ok(0) ++} ++ + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_metric(scanner: *mut LanceScanner, metric: i32) -> i32 { + scanner_poison_check!(scanner, -1); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 8805764..42f842d 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1207,6 +1207,154 @@ fn test_scanner_batch_size() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_execution_tuning_options() { ++ let (_tmp, uri) = create_multi_fragment_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(scanner, 1024) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, 64 * 1024) }, ++ 0 ++ ); ++ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 1) }, 0); ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(scanner, 1) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(scanner, 1) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_scan_in_order(scanner, false) }, ++ 0 ++ ); ++ ++ let mut ffi_stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut ffi_stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); ++ let total_rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum(); ++ assert_eq!(total_rows, 10); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ ++#[test] ++fn test_scanner_execution_tuning_options_reject_zero() { ++ let (_tmp, uri) = create_test_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(scanner, 0) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("batch_size_bytes must be greater than 0, got 0")); ++ ++ assert_eq!(unsafe { lance_scanner_set_io_buffer_size(scanner, 0) }, -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!( ++ take_last_error_message().contains("io_buffer_size_bytes must be greater than 0, got 0") ++ ); ++ ++ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 0) }, -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("batch_readahead must be greater than 0, got 0")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(scanner, 0) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("fragment_readahead must be greater than 0, got 0")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(scanner, 0) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("target_parallelism must be greater than 0, got 0")); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ ++#[test] ++fn test_scanner_execution_tuning_options_reject_after_scan_start() { ++ let (_tmp, uri) = create_test_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ let mut ffi_stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut ffi_stream) }, ++ 0 ++ ); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(scanner, 1024) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("batch_size_bytes must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, 64 * 1024) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("io_buffer_size_bytes must be set before")); ++ ++ assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("batch_readahead must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(scanner, 1) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("fragment_readahead must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(scanner, 1) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("target_parallelism must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_scan_in_order(scanner, false) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("scan_in_order must be set before")); ++ ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + // --------------------------------------------------------------------------- + // Combined filter + projection + limit + // --------------------------------------------------------------------------- +@@ -1441,6 +1589,34 @@ fn test_null_safety_comprehensive() { + unsafe { lance_scanner_set_batch_size(ptr::null_mut(), 10) }, + -1 + ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(ptr::null_mut(), 1024) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(ptr::null_mut(), 64 * 1024) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_readahead(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_fragment_readahead(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_target_parallelism(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_scan_in_order(ptr::null_mut(), true) }, ++ -1 ++ ); + assert_eq!( + unsafe { lance_scanner_with_row_id(ptr::null_mut(), true) }, + -1 +@@ -5216,6 +5392,7 @@ fn test_scanner_nearest_with_ivf_pq_index() { + 10, + ); + lance_scanner_set_nprobes(scanner, 4); ++ assert_eq!(lance_scanner_set_query_parallelism(scanner, 4), 0); + } + + let mut stream = FFI_ArrowArrayStream::empty(); +@@ -5234,6 +5411,59 @@ fn test_scanner_nearest_with_ivf_pq_index() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_query_parallelism_validation_and_lifecycle() { ++ let (_tmp, uri) = create_test_dataset(); ++ let uri_c = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, -1) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, 0) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, 2) }, ++ 0 ++ ); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, -2) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!( ++ take_last_error_message() ++ .contains("query_parallelism must be -1, 0, or greater than 0, got -2") ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_query_parallelism(scanner, 1) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("query_parallelism must be set before")); ++ ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scanner_nearest_dim_mismatch() { + let (_tmp, uri) = create_vector_dataset(64, 8); +diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp +index 17b1ab6..28762b8 100644 +--- a/tests/cpp/test_cpp_api.cpp ++++ b/tests/cpp/test_cpp_api.cpp +@@ -134,6 +134,12 @@ static void test_scanner_fluent(const std::string& uri) { + scanner.limit(5) + .offset(0) + .batch_size(2) ++ .batch_size_bytes(1024) ++ .io_buffer_size(64 * 1024) ++ .batch_readahead(1) ++ .fragment_readahead(1) ++ .target_parallelism(1) ++ .scan_in_order(false) + .statistics_callback(capture_scan_statistics, &captured); + + ArrowArrayStream stream; +@@ -370,6 +376,7 @@ static void test_nearest_smoke(const std::string& uri) { + try { + scanner.nearest("embedding", q, 8, 5) + .nprobes(2) ++ .query_parallelism(2) + .refine_factor(1) + .ef(50) + .metric(LANCE_METRIC_L2) + +From 043a1f7eac253d8ac6be3f970b60fcbf615295aa Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Sat, 5 Sep 2026 00:07:06 +0800 +Subject: [PATCH 2/2] add check + +--- + include/lance/lance.h | 8 ++++---- + include/lance/lance.hpp | 2 +- + src/scanner.rs | 11 ++++++++++- + tests/c_api_test.rs | 17 ++++++++++++++++- + 4 files changed, 31 insertions(+), 7 deletions(-) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 00f415b..541134f 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -949,12 +949,12 @@ int32_t lance_scanner_set_batch_size_bytes( + /** + * Set the scanner I/O buffer size in bytes. + * +- * The value must be greater than zero and must be set before scanning starts. +- * This bounds buffered I/O received from storage, but is not a hard limit on +- * all memory used by the scanner. ++ * The value must be between 1 and INT64_MAX, inclusive, and must be set before ++ * scanning starts. This bounds buffered I/O received from storage, but is not ++ * a hard limit on all memory used by the scanner. + * + * @param scanner Scanner handle. Must not be NULL. +- * @param io_buffer_size_bytes I/O buffer size in bytes. Must be greater than zero. ++ * @param io_buffer_size_bytes I/O buffer size in bytes, in the range [1, INT64_MAX]. + * @return 0 on success, -1 on error. + */ + int32_t lance_scanner_set_io_buffer_size( +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 3c03f86..a60dcd4 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1203,7 +1203,7 @@ class Scanner { + return *this; + } + +- /// Set the scanner I/O buffer size in bytes. ++ /// Set the scanner I/O buffer size in bytes, in the range [1, INT64_MAX]. + Scanner& io_buffer_size(uint64_t bytes) { + if (lance_scanner_set_io_buffer_size(handle_.get(), bytes) != 0) + check_error(); +diff --git a/src/scanner.rs b/src/scanner.rs +index ebedafc..6414cc0 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -837,7 +837,7 @@ unsafe fn scanner_set_batch_size_bytes_inner( + + /// Set the scanner I/O buffer size in bytes. Returns 0 on success. + /// +-/// The size must be greater than zero and must be set before the scan starts. ++/// The size must be between 1 and [`i64::MAX`] and must be set before the scan starts. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_io_buffer_size( + scanner: *mut LanceScanner, +@@ -863,6 +863,15 @@ unsafe fn scanner_set_io_buffer_size_inner( + "io_buffer_size_bytes must be greater than 0, got 0".into(), + )); + } ++ if io_buffer_size_bytes > i64::MAX as u64 { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "io_buffer_size_bytes must be at most {}, got {io_buffer_size_bytes}", ++ i64::MAX ++ ) ++ .into(), ++ )); ++ } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("io_buffer_size_bytes")?; + scanner.io_buffer_size = Some(io_buffer_size_bytes); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 42f842d..ffdd915 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1252,7 +1252,7 @@ fn test_scanner_execution_tuning_options() { + } + + #[test] +-fn test_scanner_execution_tuning_options_reject_zero() { ++fn test_scanner_execution_tuning_options_reject_invalid_values() { + let (_tmp, uri) = create_test_dataset(); + let c_uri = c_str(&uri); + let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; +@@ -1274,6 +1274,21 @@ fn test_scanner_execution_tuning_options_reject_zero() { + take_last_error_message().contains("io_buffer_size_bytes must be greater than 0, got 0") + ); + ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, i64::MAX as u64) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_io_buffer_size(scanner, u64::MAX) }, ++ -1 ++ ); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains(&format!( ++ "io_buffer_size_bytes must be at most {}, got {}", ++ i64::MAX, ++ u64::MAX ++ ))); ++ + assert_eq!(unsafe { lance_scanner_set_batch_readahead(scanner, 0) }, -1); + assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); + assert!(take_last_error_message().contains("batch_readahead must be greater than 0, got 0")); +From 057135cdd4ac6ac7b5348a1832caf7081a8541fb Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Mon, 7 Sep 2026 12:35:45 +0800 +Subject: [PATCH 1/2] feat: expose scanner use_scalar_index options + +--- + include/lance/lance.h | 83 +++++++++ + include/lance/lance.hpp | 47 ++++++ + src/scanner.rs | 335 +++++++++++++++++++++++++++++++++++++ + tests/c_api_test.rs | 296 ++++++++++++++++++++++++++++++++ + tests/cpp/test_cpp_api.cpp | 8 + + 5 files changed, 769 insertions(+) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 31213da..6ace2cb 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -147,6 +147,13 @@ typedef enum { + LANCE_METRIC_HAMMING = 3, + } LanceMetricType; + ++/** Speed / accuracy tradeoff for approximate vector search. */ ++typedef enum { ++ LANCE_APPROX_MODE_FAST = 0, ++ LANCE_APPROX_MODE_NORMAL = 1, ++ LANCE_APPROX_MODE_ACCURATE = 2, ++} LanceApproxMode; ++ + typedef enum { + LANCE_DTYPE_FLOAT32 = 0, + LANCE_DTYPE_FLOAT16 = 1, +@@ -1002,8 +1009,54 @@ int32_t lance_scanner_set_target_parallelism( + * they are ready. + */ + int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_order); ++ ++/** ++ * Configure whether scalar indices may be used to optimize filters. ++ * ++ * Scalar indices are enabled by default. Disable this to force filter ++ * evaluation without scalar indices. This setting is independent of ++ * `lance_scanner_set_use_index`, which controls vector ANN index usage. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_use_scalar_index( ++ LanceScanner* scanner, ++ bool use_scalar_index ++); ++ ++/** ++ * Configure whether row-based output batches are strict. ++ * ++ * When enabled, every batch except the last has exactly the configured row ++ * batch size. This may require copying and cannot be combined with a byte-based ++ * batch-size limit. Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_strict_batch_size( ++ LanceScanner* scanner, ++ bool strict_batch_size ++); ++ ++/** ++ * Configure whether file statistics may optimize the scan (default: true). ++ * Intended primarily for debugging and benchmarking. Must be set before ++ * scanning starts. ++ */ ++int32_t lance_scanner_set_use_stats(LanceScanner* scanner, bool use_stats); ++ + int32_t lance_scanner_with_row_id(LanceScanner* scanner, bool enable); + ++/** Include or omit the `_rowaddr` metadata column. Must be set before scanning. */ ++int32_t lance_scanner_with_row_address(LanceScanner* scanner, bool enable); ++ ++/** ++ * Configure whether deleted rows still present in storage are returned. ++ * Deleted rows have a NULL `_rowid`; callers should also enable row IDs. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_include_deleted_rows( ++ LanceScanner* scanner, ++ bool include_deleted_rows ++); ++ + /** + * Restrict scan to the given fragment IDs. Must be called before iteration. + * @param ids Array of fragment IDs +@@ -1725,6 +1778,36 @@ int32_t lance_scanner_nearest( + + int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n); + ++/** ++ * Set the minimum number of vector-index partitions to search. ++ * Must be greater than zero and no greater than `maximum_nprobes` when set. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_minimum_nprobes( ++ LanceScanner* scanner, ++ uint32_t minimum_nprobes ++); ++ ++/** ++ * Set the maximum number of vector-index partitions to search. ++ * Must be greater than zero and no less than `minimum_nprobes` when set. ++ * This only affects prefiltered searches that need more candidates. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_maximum_nprobes( ++ LanceScanner* scanner, ++ uint32_t maximum_nprobes ++); ++ ++/** ++ * Configure the speed / accuracy tradeoff for approximate vector search. ++ * Must be set before scanning starts. ++ */ ++int32_t lance_scanner_set_approx_mode( ++ LanceScanner* scanner, ++ LanceApproxMode approx_mode ++); ++ + /** + * Set vector index partition-search concurrency for each query. + * +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 404d2df..e08f76a 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1269,6 +1269,27 @@ class Scanner { + return *this; + } + ++ /// Configure whether scalar indices may be used to optimize filters. ++ Scanner& use_scalar_index(bool enable = true) { ++ if (lance_scanner_set_use_scalar_index(handle_.get(), enable) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether row-based output batches are strict. ++ Scanner& strict_batch_size(bool strict_batch_size = true) { ++ if (lance_scanner_set_strict_batch_size(handle_.get(), strict_batch_size) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether file statistics may optimize the scan. ++ Scanner& use_stats(bool use_stats = true) { ++ if (lance_scanner_set_use_stats(handle_.get(), use_stats) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Enable/disable row ID in output. + Scanner& with_row_id(bool enable = true) { + if (lance_scanner_with_row_id(handle_.get(), enable) != 0) +@@ -1276,6 +1297,20 @@ class Scanner { + return *this; + } + ++ /// Include or omit the `_rowaddr` metadata column. ++ Scanner& with_row_address(bool enable = true) { ++ if (lance_scanner_with_row_address(handle_.get(), enable) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ /// Configure whether deleted rows still present in storage are returned. ++ Scanner& include_deleted_rows(bool include_deleted_rows = true) { ++ if (lance_scanner_set_include_deleted_rows(handle_.get(), include_deleted_rows) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Restrict scan to specific fragment IDs. + Scanner& fragment_ids(const uint64_t* ids, size_t len) { + if (lance_scanner_set_fragment_ids(handle_.get(), ids, len) != 0) +@@ -1386,6 +1421,18 @@ class Scanner { + if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error(); + return *this; + } ++ Scanner& minimum_nprobes(uint32_t minimum_nprobes) { ++ if (lance_scanner_set_minimum_nprobes(handle_.get(), minimum_nprobes) != 0) check_error(); ++ return *this; ++ } ++ Scanner& maximum_nprobes(uint32_t maximum_nprobes) { ++ if (lance_scanner_set_maximum_nprobes(handle_.get(), maximum_nprobes) != 0) check_error(); ++ return *this; ++ } ++ Scanner& approx_mode(LanceApproxMode approx_mode) { ++ if (lance_scanner_set_approx_mode(handle_.get(), approx_mode) != 0) check_error(); ++ return *this; ++ } + Scanner& query_parallelism(int32_t parallelism) { + if (lance_scanner_set_query_parallelism(handle_.get(), parallelism) != 0) + check_error(); +diff --git a/src/scanner.rs b/src/scanner.rs +index d3ef3be..53cd5b6 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -21,6 +21,7 @@ use lance::dataset::scanner::{ + use lance::io::exec::fts::{FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec}; + use lance_core::Result; + use lance_index::scalar::FullTextSearchQuery; ++use lance_index::vector::ApproxMode; + use lance_io::stream::RecordBatchStream; + use lance_table::format::IndexMetadata; + use uuid::Uuid; +@@ -51,6 +52,38 @@ pub enum LanceDataType { + Int8 = 4, + } + ++/// Speed / accuracy tradeoff for approximate vector search, mirroring the C ++/// enum `LanceApproxMode`. ++#[repr(i32)] ++#[derive(Clone, Copy, Debug, PartialEq, Eq)] ++pub enum LanceApproxMode { ++ Fast = 0, ++ Normal = 1, ++ Accurate = 2, ++} ++ ++impl LanceApproxMode { ++ fn from_i32(value: i32) -> Result { ++ match value { ++ 0 => Ok(Self::Fast), ++ 1 => Ok(Self::Normal), ++ 2 => Ok(Self::Accurate), ++ _ => Err(lance_core::Error::invalid_input_source( ++ format!("approx_mode must be 0 (FAST), 1 (NORMAL), or 2 (ACCURATE), got {value}") ++ .into(), ++ )), ++ } ++ } ++ ++ fn to_approx_mode(self) -> ApproxMode { ++ match self { ++ Self::Fast => ApproxMode::Fast, ++ Self::Normal => ApproxMode::Normal, ++ Self::Accurate => ApproxMode::Accurate, ++ } ++ } ++} ++ + /// Opaque scanner handle. Stores configuration until stream materialization. + pub struct LanceScanner { + dataset: Arc, +@@ -62,16 +95,24 @@ pub struct LanceScanner { + offset: Option, + batch_size: Option, + batch_size_bytes: Option, ++ strict_batch_size: Option, + io_buffer_size: Option, + batch_readahead: Option, + fragment_readahead: Option, + target_parallelism: Option, + scan_in_order: Option, ++ use_scalar_index: Option, ++ use_stats: Option, + with_row_id: bool, ++ with_row_address: bool, ++ include_deleted_rows: bool, + fragment_ids: Option>, + index_segments: Option>, + nearest: Option, + nprobes: Option, ++ minimum_nprobes: Option, ++ maximum_nprobes: Option, ++ approx_mode: Option, + query_parallelism: Option, + refine_factor: Option, + ef: Option, +@@ -138,16 +179,24 @@ impl LanceScanner { + offset: None, + batch_size: None, + batch_size_bytes: None, ++ strict_batch_size: None, + io_buffer_size: None, + batch_readahead: None, + fragment_readahead: None, + target_parallelism: None, + scan_in_order: None, ++ use_scalar_index: None, ++ use_stats: None, + with_row_id: false, ++ with_row_address: false, ++ include_deleted_rows: false, + fragment_ids: None, + index_segments: None, + nearest: None, + nprobes: None, ++ minimum_nprobes: None, ++ maximum_nprobes: None, ++ approx_mode: None, + query_parallelism: None, + refine_factor: None, + ef: None, +@@ -258,6 +307,9 @@ impl LanceScanner { + if let Some(batch_size_bytes) = self.batch_size_bytes { + scanner.batch_size_bytes(batch_size_bytes); + } ++ if let Some(strict_batch_size) = self.strict_batch_size { ++ scanner.strict_batch_size(strict_batch_size); ++ } + if let Some(io_buffer_size) = self.io_buffer_size { + scanner.io_buffer_size(io_buffer_size); + } +@@ -273,9 +325,21 @@ impl LanceScanner { + if let Some(scan_in_order) = self.scan_in_order { + scanner.scan_in_order(scan_in_order); + } ++ if let Some(use_scalar_index) = self.use_scalar_index { ++ scanner.use_scalar_index(use_scalar_index); ++ } ++ if let Some(use_stats) = self.use_stats { ++ scanner.use_stats(use_stats); ++ } + if self.with_row_id { + scanner.with_row_id(); + } ++ if self.with_row_address { ++ scanner.with_row_address(); ++ } ++ if self.include_deleted_rows { ++ scanner.include_deleted_rows(); ++ } + self.apply_fragment_filter(&mut scanner)?; + if self.index_segments.is_some() && self.nearest.is_none() { + return Err(lance_core::Error::invalid_input_source( +@@ -302,6 +366,15 @@ impl LanceScanner { + if let Some(np) = self.nprobes { + scanner.nprobes(np as usize); + } ++ if let Some(minimum_nprobes) = self.minimum_nprobes { ++ scanner.minimum_nprobes(minimum_nprobes as usize); ++ } ++ if let Some(maximum_nprobes) = self.maximum_nprobes { ++ scanner.maximum_nprobes(maximum_nprobes as usize); ++ } ++ if let Some(approx_mode) = self.approx_mode { ++ scanner.approx_mode(approx_mode.to_approx_mode()); ++ } + if let Some(query_parallelism) = self.query_parallelism { + scanner.query_parallelism(query_parallelism); + } +@@ -1035,6 +1108,92 @@ unsafe fn scanner_set_scan_in_order_inner( + Ok(0) + } + ++/// Configure whether scalar indices may be used to optimize filters. ++/// ++/// Scalar indices are enabled by default in Lance. Must be set before the scan ++/// starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_use_scalar_index( ++ scanner: *mut LanceScanner, ++ use_scalar_index: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_use_scalar_index_inner(scanner, use_scalar_index) ++ }) ++} ++ ++unsafe fn scanner_set_use_scalar_index_inner( ++ scanner: *mut LanceScanner, ++ use_scalar_index: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("use_scalar_index")?; ++ scanner.use_scalar_index = Some(use_scalar_index); ++ Ok(0) ++} ++ ++/// Configure whether output batches use the exact row-based batch size. ++/// ++/// Must be set before the scan starts. Lance rejects enabling this together ++/// with a byte-based batch-size limit when the scan is materialized. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_strict_batch_size( ++ scanner: *mut LanceScanner, ++ strict_batch_size: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_strict_batch_size_inner(scanner, strict_batch_size) ++ }) ++} ++ ++unsafe fn scanner_set_strict_batch_size_inner( ++ scanner: *mut LanceScanner, ++ strict_batch_size: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("strict_batch_size")?; ++ scanner.strict_batch_size = Some(strict_batch_size); ++ Ok(0) ++} ++ ++/// Configure whether file statistics may be used to optimize the scan. ++/// ++/// Statistics are enabled by default. Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_use_stats( ++ scanner: *mut LanceScanner, ++ use_stats: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_use_stats_inner(scanner, use_stats) ++ }) ++} ++ ++unsafe fn scanner_set_use_stats_inner(scanner: *mut LanceScanner, use_stats: bool) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("use_stats")?; ++ scanner.use_stats = Some(use_stats); ++ Ok(0) ++} ++ + /// Enable or disable row ID in scan output. Returns 0. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_with_row_id( +@@ -1058,6 +1217,62 @@ unsafe fn scanner_with_row_id_inner(scanner: *mut LanceScanner, enable: bool) -> + Ok(0) + } + ++/// Enable or disable the `_rowaddr` metadata column in scan output. ++/// ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_with_row_address( ++ scanner: *mut LanceScanner, ++ enable: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_with_row_address_inner(scanner, enable) ++ }) ++} ++ ++unsafe fn scanner_with_row_address_inner(scanner: *mut LanceScanner, enable: bool) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("with_row_address")?; ++ scanner.with_row_address = enable; ++ Ok(0) ++} ++ ++/// Configure whether deleted rows still present in storage are returned. ++/// ++/// Deleted rows have a NULL `_rowid`, so callers should also enable row IDs. ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_include_deleted_rows( ++ scanner: *mut LanceScanner, ++ include_deleted_rows: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_include_deleted_rows_inner(scanner, include_deleted_rows) ++ }) ++} ++ ++unsafe fn scanner_set_include_deleted_rows_inner( ++ scanner: *mut LanceScanner, ++ include_deleted_rows: bool, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("include_deleted_rows")?; ++ scanner.include_deleted_rows = include_deleted_rows; ++ Ok(0) ++} ++ + /// Restrict the scan to the given fragment IDs. + /// Must be called before any iteration method. + /// +@@ -2044,6 +2259,126 @@ scanner_set_u32!(lance_scanner_set_nprobes, nprobes); + scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor); + scanner_set_u32!(lance_scanner_set_ef, ef); + ++/// Set the minimum number of vector-index partitions to search. ++/// ++/// The value must be greater than zero, no greater than a configured ++/// `maximum_nprobes`, and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_minimum_nprobes( ++ scanner: *mut LanceScanner, ++ minimum_nprobes: u32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_minimum_nprobes_inner(scanner, minimum_nprobes) ++ }) ++} ++ ++unsafe fn scanner_set_minimum_nprobes_inner( ++ scanner: *mut LanceScanner, ++ minimum_nprobes: u32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if minimum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "minimum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("minimum_nprobes")?; ++ if let Some(maximum_nprobes) = scanner.maximum_nprobes ++ && minimum_nprobes > maximum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ scanner.minimum_nprobes = Some(minimum_nprobes); ++ Ok(0) ++} ++ ++/// Set the maximum number of vector-index partitions to search. ++/// ++/// The value must be greater than zero, no less than a configured ++/// `minimum_nprobes`, and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_maximum_nprobes( ++ scanner: *mut LanceScanner, ++ maximum_nprobes: u32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_maximum_nprobes_inner(scanner, maximum_nprobes) ++ }) ++} ++ ++unsafe fn scanner_set_maximum_nprobes_inner( ++ scanner: *mut LanceScanner, ++ maximum_nprobes: u32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ if maximum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "maximum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("maximum_nprobes")?; ++ if let Some(minimum_nprobes) = scanner.minimum_nprobes ++ && maximum_nprobes < minimum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ scanner.maximum_nprobes = Some(maximum_nprobes); ++ Ok(0) ++} ++ ++/// Configure the speed / accuracy tradeoff for approximate vector search. ++/// ++/// Must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_approx_mode( ++ scanner: *mut LanceScanner, ++ approx_mode: i32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_approx_mode_inner(scanner, approx_mode) ++ }) ++} ++ ++unsafe fn scanner_set_approx_mode_inner( ++ scanner: *mut LanceScanner, ++ approx_mode: i32, ++) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let approx_mode = LanceApproxMode::from_i32(approx_mode)?; ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("approx_mode")?; ++ scanner.approx_mode = Some(approx_mode); ++ Ok(0) ++} ++ + /// Set vector index partition-search concurrency for each query. + /// + /// `-1` uses the CPU pool size, `0` selects Lance's automatic policy, and +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index bde742d..a550a93 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1237,6 +1237,12 @@ fn test_scanner_execution_tuning_options() { + unsafe { lance_scanner_set_scan_in_order(scanner, false) }, + 0 + ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(scanner, false) }, ++ 0 ++ ); ++ assert_eq!(unsafe { lance_scanner_set_use_stats(scanner, false) }, 0); ++ assert_eq!(unsafe { lance_scanner_with_row_address(scanner, true) }, 0); + + let mut ffi_stream = FFI_ArrowArrayStream::empty(); + assert_eq!( +@@ -1244,6 +1250,7 @@ fn test_scanner_execution_tuning_options() { + 0 + ); + let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); ++ assert!(reader.schema().field_with_name("_rowaddr").is_ok()); + let total_rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum(); + assert_eq!(total_rows, 10); + +@@ -1360,6 +1367,30 @@ fn test_scanner_execution_tuning_options_reject_after_scan_start() { + ); + assert!(take_last_error_message().contains("scan_in_order must be set before")); + ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(scanner, false) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("use_scalar_index must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(scanner, true) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("strict_batch_size must be set before")); ++ ++ assert_eq!(unsafe { lance_scanner_set_use_stats(scanner, false) }, -1); ++ assert!(take_last_error_message().contains("use_stats must be set before")); ++ ++ assert_eq!(unsafe { lance_scanner_with_row_address(scanner, true) }, -1); ++ assert!(take_last_error_message().contains("with_row_address must be set before")); ++ ++ assert_eq!( ++ unsafe { lance_scanner_set_include_deleted_rows(scanner, true) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("include_deleted_rows must be set before")); ++ + let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut ffi_stream) }.unwrap(); + assert_eq!( + reader.map(|batch| batch.unwrap().num_rows()).sum::(), +@@ -1370,6 +1401,79 @@ fn test_scanner_execution_tuning_options_reject_after_scan_start() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_strict_batch_size_across_fragments() { ++ let (_tmp, uri) = create_multi_fragment_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!(unsafe { lance_scanner_set_batch_size(scanner, 3) }, 0); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(scanner, true) }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ let batch_sizes = reader ++ .map(|batch| batch.unwrap().num_rows()) ++ .collect::>(); ++ assert_eq!(batch_sizes, vec![3, 3, 3, 1]); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ ++#[test] ++fn test_scanner_include_deleted_rows() { ++ let (_tmp, uri) = create_multi_fragment_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let predicate = c_str("id >= 8"); ++ let mut num_deleted = 0; ++ assert_eq!( ++ unsafe { lance_dataset_delete(ds, predicate.as_ptr(), &mut num_deleted) }, ++ 0 ++ ); ++ assert_eq!(num_deleted, 2); ++ ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!(unsafe { lance_scanner_with_row_id(scanner, true) }, 0); ++ assert_eq!( ++ unsafe { lance_scanner_set_include_deleted_rows(scanner, true) }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ let batches = reader.map(|batch| batch.unwrap()).collect::>(); ++ assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 10); ++ assert_eq!( ++ batches ++ .iter() ++ .map(|batch| batch.column_by_name("_rowid").unwrap().null_count()) ++ .sum::(), ++ 2 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + // --------------------------------------------------------------------------- + // Combined filter + projection + limit + // --------------------------------------------------------------------------- +@@ -1632,10 +1736,42 @@ fn test_null_safety_comprehensive() { + unsafe { lance_scanner_set_scan_in_order(ptr::null_mut(), true) }, + -1 + ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(ptr::null_mut(), false) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(ptr::null_mut(), true) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_stats(ptr::null_mut(), false) }, ++ -1 ++ ); + assert_eq!( + unsafe { lance_scanner_with_row_id(ptr::null_mut(), true) }, + -1 + ); ++ assert_eq!( ++ unsafe { lance_scanner_with_row_address(ptr::null_mut(), true) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_include_deleted_rows(ptr::null_mut(), true) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_minimum_nprobes(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_maximum_nprobes(ptr::null_mut(), 1) }, ++ -1 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_approx_mode(ptr::null_mut(), LanceApproxMode::Normal as i32,) }, ++ -1 ++ ); + + // Scanner iteration with NULL. + let mut ffi_stream2 = FFI_ArrowArrayStream::empty(); +@@ -3077,6 +3213,90 @@ fn test_create_scalar_index_btree() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_set_use_scalar_index_controls_filter_planning() { ++ let (_tmp, uri) = create_test_dataset(); ++ let uri_c = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let column = c_str("id"); ++ assert_eq!( ++ unsafe { ++ lance_dataset_create_scalar_index( ++ ds, ++ column.as_ptr(), ++ ptr::null(), ++ LanceScalarIndexType::BTree as i32, ++ ptr::null(), ++ false, ++ ) ++ }, ++ 0 ++ ); ++ ++ let run_scan = |use_scalar_index: bool| { ++ let filter = c_str("id = 3"); ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), filter.as_ptr()) }; ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_scalar_index(scanner, use_scalar_index) }, ++ 0 ++ ); ++ ++ let mut captured = CapturedScanStatistics::default(); ++ assert_eq!( ++ unsafe { ++ lance_scanner_set_statistics_callback( ++ scanner, ++ Some(capture_scan_statistics), ++ (&mut captured as *mut CapturedScanStatistics).cast(), ++ ) ++ }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ let ids = reader ++ .flat_map(|batch| { ++ let batch = batch.unwrap(); ++ batch ++ .column_by_name("id") ++ .unwrap() ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values() ++ .to_vec() ++ }) ++ .collect::>(); ++ ++ assert_eq!(captured.calls, 1); ++ unsafe { lance_scanner_close(scanner) }; ++ (ids, captured) ++ }; ++ ++ let (indexed_ids, indexed_statistics) = run_scan(true); ++ let (unindexed_ids, unindexed_statistics) = run_scan(false); ++ assert_eq!(indexed_ids, vec![3]); ++ assert_eq!(unindexed_ids, indexed_ids); ++ assert!( ++ indexed_statistics.indices_loaded > 0, ++ "enabled scan should load the scalar index" ++ ); ++ assert_eq!( ++ unindexed_statistics.indices_loaded, 0, ++ "disabled scan should bypass the scalar index" ++ ); ++ ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scalar_index_segment_build_is_fragment_scoped_and_uncommitted() { + let (_tmp, uri) = create_many_small_fragments(2); +@@ -5409,6 +5629,12 @@ fn test_scanner_nearest_with_ivf_pq_index() { + 10, + ); + lance_scanner_set_nprobes(scanner, 4); ++ assert_eq!(lance_scanner_set_minimum_nprobes(scanner, 2), 0); ++ assert_eq!(lance_scanner_set_maximum_nprobes(scanner, 6), 0); ++ assert_eq!( ++ lance_scanner_set_approx_mode(scanner, LanceApproxMode::Accurate as i32), ++ 0 ++ ); + assert_eq!(lance_scanner_set_query_parallelism(scanner, 4), 0); + } + +@@ -5428,6 +5654,76 @@ fn test_scanner_nearest_with_ivf_pq_index() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() { ++ let (_tmp, uri) = create_test_dataset(); ++ let uri_c = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert!(!scanner.is_null()); ++ ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 0) }, -1); ++ assert!(take_last_error_message().contains("minimum_nprobes must be greater than 0, got 0")); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 0) }, -1); ++ assert!(take_last_error_message().contains("maximum_nprobes must be greater than 0, got 0")); ++ assert_eq!(unsafe { lance_scanner_set_approx_mode(scanner, 3) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("approx_mode must be 0 (FAST), 1 (NORMAL), or 2 (ACCURATE), got 3") ++ ); ++ ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 3) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("minimum_nprobes (3) must not exceed maximum_nprobes (2)") ++ ); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("minimum_nprobes (2) must not exceed maximum_nprobes (1)") ++ ); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1); ++ assert!( ++ take_last_error_message() ++ .contains("maximum_nprobes (1) must not be less than minimum_nprobes (2)") ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_approx_mode(scanner, LanceApproxMode::Fast as i32) }, ++ 0 ++ ); ++ ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("minimum_nprobes must be set before")); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("maximum_nprobes must be set before")); ++ assert_eq!( ++ unsafe { lance_scanner_set_approx_mode(scanner, LanceApproxMode::Normal as i32) }, ++ -1 ++ ); ++ assert!(take_last_error_message().contains("approx_mode must be set before")); ++ ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ ++ unsafe { lance_scanner_close(scanner) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scanner_query_parallelism_validation_and_lifecycle() { + let (_tmp, uri) = create_test_dataset(); +diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp +index 28762b8..60b8fc4 100644 +--- a/tests/cpp/test_cpp_api.cpp ++++ b/tests/cpp/test_cpp_api.cpp +@@ -140,6 +140,11 @@ static void test_scanner_fluent(const std::string& uri) { + .fragment_readahead(1) + .target_parallelism(1) + .scan_in_order(false) ++ .use_scalar_index(false) ++ .strict_batch_size(false) ++ .use_stats(false) ++ .with_row_address(true) ++ .include_deleted_rows(false) + .statistics_callback(capture_scan_statistics, &captured); + + ArrowArrayStream stream; +@@ -376,6 +381,9 @@ static void test_nearest_smoke(const std::string& uri) { + try { + scanner.nearest("embedding", q, 8, 5) + .nprobes(2) ++ .minimum_nprobes(1) ++ .maximum_nprobes(2) ++ .approx_mode(LANCE_APPROX_MODE_NORMAL) + .query_parallelism(2) + .refine_factor(1) + .ef(50) + +From e894f591aef358cd36fdbf915c4d5b95fd0e8348 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Mon, 7 Sep 2026 14:27:10 +0800 +Subject: [PATCH 2/2] update + +--- + include/lance/lance.h | 18 +++- + include/lance/lance.hpp | 7 +- + src/scanner.rs | 210 +++++++++++++++++++++++++++++++--------- + tests/c_api_test.rs | 77 +++++++++++++++ + 4 files changed, 261 insertions(+), 51 deletions(-) + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 6ace2cb..8173ae5 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -946,7 +946,8 @@ int32_t lance_scanner_set_batch_size(LanceScanner* scanner, int64_t batch_size); + * Set the target output batch size in bytes. + * + * When set, this takes precedence over the row-based batch size. The value +- * must be greater than zero and must be set before scanning starts. ++ * must be greater than zero and must be set before scanning starts. The call ++ * is rejected without changing scanner state if strict batch sizing is enabled. + */ + int32_t lance_scanner_set_batch_size_bytes( + LanceScanner* scanner, +@@ -1028,7 +1029,8 @@ int32_t lance_scanner_set_use_scalar_index( + * + * When enabled, every batch except the last has exactly the configured row + * batch size. This may require copying and cannot be combined with a byte-based +- * batch-size limit. Must be set before scanning starts. ++ * batch-size limit. The call is rejected without changing scanner state if a ++ * byte limit is already set. Must be set before scanning starts. + */ + int32_t lance_scanner_set_strict_batch_size( + LanceScanner* scanner, +@@ -1776,11 +1778,19 @@ int32_t lance_scanner_nearest( + uint32_t k + ); + +-int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t n); ++/** ++ * Set both the minimum and maximum vector-index partition-search bounds. ++ * ++ * This replaces both bounds configured by earlier calls to any nprobes ++ * setter. The value must be greater than zero. Must be set before scanning. ++ */ ++int32_t lance_scanner_set_nprobes(LanceScanner* scanner, uint32_t nprobes); + + /** + * Set the minimum number of vector-index partitions to search. ++ * This replaces only the minimum bound; the current maximum is preserved. + * Must be greater than zero and no greater than `maximum_nprobes` when set. ++ * An invalid resulting range is rejected without changing either bound. + * Must be set before scanning starts. + */ + int32_t lance_scanner_set_minimum_nprobes( +@@ -1790,8 +1800,10 @@ int32_t lance_scanner_set_minimum_nprobes( + + /** + * Set the maximum number of vector-index partitions to search. ++ * This replaces only the maximum bound; the current minimum is preserved. + * Must be greater than zero and no less than `minimum_nprobes` when set. + * This only affects prefiltered searches that need more candidates. ++ * An invalid resulting range is rejected without changing either bound. + * Must be set before scanning starts. + */ + int32_t lance_scanner_set_maximum_nprobes( +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index e08f76a..c12c0c6 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1417,14 +1417,17 @@ class Scanner { + return *this; + } + +- Scanner& nprobes(uint32_t n) { +- if (lance_scanner_set_nprobes(handle_.get(), n) != 0) check_error(); ++ /// Replace both minimum and maximum partition-search bounds. ++ Scanner& nprobes(uint32_t nprobes) { ++ if (lance_scanner_set_nprobes(handle_.get(), nprobes) != 0) check_error(); + return *this; + } ++ /// Replace only the minimum partition-search bound. + Scanner& minimum_nprobes(uint32_t minimum_nprobes) { + if (lance_scanner_set_minimum_nprobes(handle_.get(), minimum_nprobes) != 0) check_error(); + return *this; + } ++ /// Replace only the maximum partition-search bound. + Scanner& maximum_nprobes(uint32_t maximum_nprobes) { + if (lance_scanner_set_maximum_nprobes(handle_.get(), maximum_nprobes) != 0) check_error(); + return *this; +diff --git a/src/scanner.rs b/src/scanner.rs +index 53cd5b6..4ceeb0e 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -109,9 +109,7 @@ pub struct LanceScanner { + fragment_ids: Option>, + index_segments: Option>, + nearest: Option, +- nprobes: Option, +- minimum_nprobes: Option, +- maximum_nprobes: Option, ++ nprobes: NprobesRange, + approx_mode: Option, + query_parallelism: Option, + refine_factor: Option, +@@ -148,6 +146,72 @@ struct NearestQuery { + k: u32, + } + ++/// The effective adaptive partition-search range shared by all three nprobes ++/// setters. Updates are computed and validated before replacing this state. ++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] ++struct NprobesRange { ++ minimum: Option, ++ maximum: Option, ++} ++ ++impl NprobesRange { ++ fn exact(nprobes: u32) -> Result { ++ if nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ Ok(Self { ++ minimum: Some(nprobes), ++ maximum: Some(nprobes), ++ }) ++ } ++ ++ fn with_minimum(self, minimum_nprobes: u32) -> Result { ++ if minimum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "minimum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ if let Some(maximum_nprobes) = self.maximum ++ && minimum_nprobes > maximum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ Ok(Self { ++ minimum: Some(minimum_nprobes), ++ ..self ++ }) ++ } ++ ++ fn with_maximum(self, maximum_nprobes: u32) -> Result { ++ if maximum_nprobes == 0 { ++ return Err(lance_core::Error::invalid_input_source( ++ "maximum_nprobes must be greater than 0, got 0".into(), ++ )); ++ } ++ if let Some(minimum_nprobes) = self.minimum ++ && maximum_nprobes < minimum_nprobes ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})" ++ ) ++ .into(), ++ )); ++ } ++ Ok(Self { ++ maximum: Some(maximum_nprobes), ++ ..self ++ }) ++ } ++} ++ + /// Poll status for `lance_scanner_poll_next`. + #[repr(C)] + #[derive(Debug, PartialEq, Eq)] +@@ -193,9 +257,7 @@ impl LanceScanner { + fragment_ids: None, + index_segments: None, + nearest: None, +- nprobes: None, +- minimum_nprobes: None, +- maximum_nprobes: None, ++ nprobes: NprobesRange::default(), + approx_mode: None, + query_parallelism: None, + refine_factor: None, +@@ -363,13 +425,10 @@ impl LanceScanner { + } + if let Some(n) = &self.nearest { + scanner.nearest(&n.column, n.query.as_ref(), n.k as usize)?; +- if let Some(np) = self.nprobes { +- scanner.nprobes(np as usize); +- } +- if let Some(minimum_nprobes) = self.minimum_nprobes { ++ if let Some(minimum_nprobes) = self.nprobes.minimum { + scanner.minimum_nprobes(minimum_nprobes as usize); + } +- if let Some(maximum_nprobes) = self.maximum_nprobes { ++ if let Some(maximum_nprobes) = self.nprobes.maximum { + scanner.maximum_nprobes(maximum_nprobes as usize); + } + if let Some(approx_mode) = self.approx_mode { +@@ -930,6 +989,14 @@ unsafe fn scanner_set_batch_size_bytes_inner( + } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("batch_size_bytes")?; ++ if scanner.strict_batch_size == Some(true) { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}" ++ ) ++ .into(), ++ )); ++ } + scanner.batch_size_bytes = Some(batch_size_bytes); + Ok(0) + } +@@ -1140,8 +1207,8 @@ unsafe fn scanner_set_use_scalar_index_inner( + + /// Configure whether output batches use the exact row-based batch size. + /// +-/// Must be set before the scan starts. Lance rejects enabling this together +-/// with a byte-based batch-size limit when the scan is materialized. ++/// Must be set before the scan starts. Enabling this together with a ++/// byte-based batch-size limit is rejected without changing scanner state. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_strict_batch_size( + scanner: *mut LanceScanner, +@@ -1164,6 +1231,14 @@ unsafe fn scanner_set_strict_batch_size_inner( + } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("strict_batch_size")?; ++ if strict_batch_size && let Some(batch_size_bytes) = scanner.batch_size_bytes { ++ return Err(lance_core::Error::invalid_input_source( ++ format!( ++ "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}" ++ ) ++ .into(), ++ )); ++ } + scanner.strict_batch_size = Some(strict_batch_size); + Ok(0) + } +@@ -2255,10 +2330,38 @@ macro_rules! scanner_set_u32 { + }; + } + +-scanner_set_u32!(lance_scanner_set_nprobes, nprobes); + scanner_set_u32!(lance_scanner_set_refine_factor, refine_factor); + scanner_set_u32!(lance_scanner_set_ef, ef); + ++/// Set both vector-index partition-search bounds to the same value. ++/// ++/// This replaces any values previously configured through ++/// `minimum_nprobes` or `maximum_nprobes`. The value must be greater than zero ++/// and must be set before the scan starts. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_nprobes( ++ scanner: *mut LanceScanner, ++ nprobes: u32, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, unsafe { ++ scanner_set_nprobes_inner(scanner, nprobes) ++ }) ++} ++ ++unsafe fn scanner_set_nprobes_inner(scanner: *mut LanceScanner, nprobes: u32) -> Result { ++ if scanner.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "scanner is NULL".into(), ++ )); ++ } ++ let scanner = unsafe { &mut *scanner }; ++ scanner.ensure_scan_not_started("nprobes")?; ++ let next = NprobesRange::exact(nprobes)?; ++ scanner.nprobes = next; ++ Ok(0) ++} ++ + /// Set the minimum number of vector-index partitions to search. + /// + /// The value must be greater than zero, no greater than a configured +@@ -2283,24 +2386,10 @@ unsafe fn scanner_set_minimum_nprobes_inner( + "scanner is NULL".into(), + )); + } +- if minimum_nprobes == 0 { +- return Err(lance_core::Error::invalid_input_source( +- "minimum_nprobes must be greater than 0, got 0".into(), +- )); +- } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("minimum_nprobes")?; +- if let Some(maximum_nprobes) = scanner.maximum_nprobes +- && minimum_nprobes > maximum_nprobes +- { +- return Err(lance_core::Error::invalid_input_source( +- format!( +- "minimum_nprobes ({minimum_nprobes}) must not exceed maximum_nprobes ({maximum_nprobes})" +- ) +- .into(), +- )); +- } +- scanner.minimum_nprobes = Some(minimum_nprobes); ++ let next = scanner.nprobes.with_minimum(minimum_nprobes)?; ++ scanner.nprobes = next; + Ok(0) + } + +@@ -2328,24 +2417,10 @@ unsafe fn scanner_set_maximum_nprobes_inner( + "scanner is NULL".into(), + )); + } +- if maximum_nprobes == 0 { +- return Err(lance_core::Error::invalid_input_source( +- "maximum_nprobes must be greater than 0, got 0".into(), +- )); +- } + let scanner = unsafe { &mut *scanner }; + scanner.ensure_scan_not_started("maximum_nprobes")?; +- if let Some(minimum_nprobes) = scanner.minimum_nprobes +- && maximum_nprobes < minimum_nprobes +- { +- return Err(lance_core::Error::invalid_input_source( +- format!( +- "maximum_nprobes ({maximum_nprobes}) must not be less than minimum_nprobes ({minimum_nprobes})" +- ) +- .into(), +- )); +- } +- scanner.maximum_nprobes = Some(maximum_nprobes); ++ let next = scanner.nprobes.with_maximum(maximum_nprobes)?; ++ scanner.nprobes = next; + Ok(0) + } + +@@ -2885,6 +2960,49 @@ mod tests { + ) + } + ++ #[test] ++ fn nprobes_setters_share_one_validated_range() { ++ let (_tmp, uri) = create_test_dataset(); ++ let (dataset, scanner) = open_dataset_and_scanner(&uri); ++ let assert_range = |minimum, maximum| { ++ assert_eq!( ++ unsafe { &*scanner }.nprobes, ++ NprobesRange { minimum, maximum } ++ ); ++ }; ++ ++ assert_range(None, None); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 2) }, 0); ++ assert_range(Some(2), None); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 5) }, 0); ++ assert_range(Some(2), Some(5)); ++ ++ // The combined setter replaces both bounds. ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 4) }, 0); ++ assert_range(Some(4), Some(4)); ++ ++ // A failed partial update leaves both bounds unchanged. ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 5) }, -1); ++ assert_range(Some(4), Some(4)); ++ ++ // Widening the maximum first makes the new minimum valid. ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 6) }, 0); ++ assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 5) }, 0); ++ assert_range(Some(5), Some(6)); ++ ++ // A later combined call deterministically replaces the widened range. ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 3) }, 0); ++ assert_range(Some(3), Some(3)); ++ assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 2) }, -1); ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 0) }, -1); ++ assert_range(Some(3), Some(3)); ++ ++ unsafe { ++ lance_scanner_close(scanner); ++ lance_dataset_close(dataset); ++ } ++ } ++ + #[test] + fn prepared_fts_index_only_plan_does_not_scan_indexed_fragment_row_ids() { + let (_tmp, uri) = create_test_dataset(); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index a550a93..3b3424b 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -1318,6 +1318,78 @@ fn test_scanner_execution_tuning_options_reject_invalid_values() { + unsafe { lance_dataset_close(ds) }; + } + ++#[test] ++fn test_scanner_strict_batch_size_and_bytes_conflict_is_recoverable() { ++ let (_tmp, uri) = create_test_dataset(); ++ let c_uri = c_str(&uri); ++ let ds = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!ds.is_null()); ++ ++ let consume = |scanner| { ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!( ++ unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream) }, ++ 0 ++ ); ++ let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream) }.unwrap(); ++ assert_eq!( ++ reader.map(|batch| batch.unwrap().num_rows()).sum::(), ++ 5 ++ ); ++ }; ++ ++ // A byte limit already exists: strict=true is rejected without starting ++ // the scan or replacing the prior strict setting. ++ let bytes_first = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(bytes_first, 1024) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(bytes_first, true) }, ++ -1 ++ ); ++ assert!( ++ take_last_error_message() ++ .contains("strict_batch_size=true cannot be combined with batch_size_bytes=1024") ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_use_stats(bytes_first, false) }, ++ 0, ++ "the rejected setter must not mark the scan as started" ++ ); ++ consume(bytes_first); ++ ++ // Strict sizing already exists: the byte limit is rejected without ++ // mutation. The caller can disable strict sizing and retry on this handle. ++ let strict_first = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(strict_first, true) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(strict_first, 1024) }, ++ -1 ++ ); ++ assert!( ++ take_last_error_message() ++ .contains("strict_batch_size=true cannot be combined with batch_size_bytes=1024") ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_strict_batch_size(strict_first, false) }, ++ 0 ++ ); ++ assert_eq!( ++ unsafe { lance_scanner_set_batch_size_bytes(strict_first, 1024) }, ++ 0 ++ ); ++ consume(strict_first); ++ ++ unsafe { lance_scanner_close(bytes_first) }; ++ unsafe { lance_scanner_close(strict_first) }; ++ unsafe { lance_dataset_close(ds) }; ++} ++ + #[test] + fn test_scanner_execution_tuning_options_reject_after_scan_start() { + let (_tmp, uri) = create_test_dataset(); +@@ -1760,6 +1832,7 @@ fn test_null_safety_comprehensive() { + unsafe { lance_scanner_set_include_deleted_rows(ptr::null_mut(), true) }, + -1 + ); ++ assert_eq!(unsafe { lance_scanner_set_nprobes(ptr::null_mut(), 1) }, -1); + assert_eq!( + unsafe { lance_scanner_set_minimum_nprobes(ptr::null_mut(), 1) }, + -1 +@@ -5663,6 +5736,8 @@ fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() { + let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; + assert!(!scanner.is_null()); + ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 0) }, -1); ++ assert!(take_last_error_message().contains("nprobes must be greater than 0, got 0")); + assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 0) }, -1); + assert!(take_last_error_message().contains("minimum_nprobes must be greater than 0, got 0")); + assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 0) }, -1); +@@ -5704,6 +5779,8 @@ fn test_scanner_adaptive_nprobes_and_approx_mode_validation_and_lifecycle() { + 0 + ); + ++ assert_eq!(unsafe { lance_scanner_set_nprobes(scanner, 1) }, -1); ++ assert!(take_last_error_message().contains("nprobes must be set before")); + assert_eq!(unsafe { lance_scanner_set_minimum_nprobes(scanner, 1) }, -1); + assert!(take_last_error_message().contains("minimum_nprobes must be set before")); + assert_eq!(unsafe { lance_scanner_set_maximum_nprobes(scanner, 1) }, -1); diff --git a/thirdparty/patches/lance-c-0.1.9-pr-77.patch b/thirdparty/patches/lance-c-0.1.9-pr-77.patch new file mode 100644 index 00000000000000..341a1c692dcebd --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-77.patch @@ -0,0 +1,1863 @@ +From eaf06c0374e62de6b519f55efe17de0c58e88c0a Mon Sep 17 00:00:00 2001 +From: "jianjian.xie" +Date: Fri, 4 Sep 2026 22:04:55 -0700 +Subject: [PATCH] build: bump lance to v11.0.0 + +Summary: +Intent: +- Move the lance git pins from e934cc2c to ab6b5bbe (lance v11.0.0 release tag) + so lance-c tracks a released upstream version instead of an arbitrary commit. +- Pick up the v11 blob APIs (read_blob_ranges, Option-based take_blobs results) + needed to answer #76 without a second pin bump. + +Changes: +- Point all lance, lance-core, lance-file, lance-index, lance-io, lance-linalg, + lance-table, lance-datafusion, and lance-datagen dependencies at ab6b5bbe. +- Re-resolve Cargo.lock; blake3, jiff, and reqwest 0.13 were unlocked explicitly + because v11 raised their minimum versions, the rest follows from lance v11 + (opendal 0.58, lance-namespace-reqwest-client 0.11, etc.). +- Adapt to upstream signature changes: build_global_bm25_scorer takes an optional + metrics collector, MatchQueryExec/PhraseQueryExec::new_with_segments are now + fallible, DataFile::new takes a ConcreteFileVersion, and pb::IndexMetadata + gained covering_fields. +- Keep the DOT PQ strict-subset guard; upstream make_global_pq is unchanged at + v11, so only the referenced revision in the comment and error text moved. + +Test Plan: +- cargo fmt, cargo check --all-targets, cargo clippy --all-targets -D warnings. +- cargo test: 367 passed, 0 failed, 2 ignored. +- cargo test --test compile_and_run_test -- --ignored: 2 passed (C and C++ + compile-and-run against the rebuilt library). + +Co-Authored-By: Claude Fable 5.1 +--- + Cargo.lock | 653 ++++++++++++++++++++++++------------------- + Cargo.toml | 22 +- + src/fts_query.rs | 4 +- + src/index_segment.rs | 4 +- + src/scanner.rs | 4 +- + tests/c_api_test.rs | 8 +- + 6 files changed, 391 insertions(+), 304 deletions(-) + +diff --git a/Cargo.lock b/Cargo.lock +index 60c1caf..bc37cb9 100644 +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -222,7 +222,7 @@ dependencies = [ + "arrow-schema", + "arrow-select", + "atoi", +- "base64", ++ "base64 0.22.1", + "chrono", + "comfy-table", + "half", +@@ -332,7 +332,7 @@ version = "58.3.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "serde_core", + "serde_json", + ] +@@ -434,6 +434,16 @@ dependencies = [ + "loom", + ] + ++[[package]] ++name = "asyncband" ++version = "0.6.7" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" ++dependencies = [ ++ "hashbrown 0.17.1", ++ "slab", ++] ++ + [[package]] + name = "atoi" + version = "2.0.0" +@@ -504,7 +514,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" + dependencies = [ + "aws-lc-sys", +- "untrusted 0.7.1", + "zeroize", + ] + +@@ -631,7 +640,7 @@ dependencies = [ + "bytes", + "form_urlencoded", + "hex", +- "hmac", ++ "hmac 0.12.1", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", +@@ -829,6 +838,12 @@ version = "0.22.1" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + ++[[package]] ++name = "base64" ++version = "0.23.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" ++ + [[package]] + name = "base64-simd" + version = "0.8.0" +@@ -858,6 +873,12 @@ dependencies = [ + "num-traits", + ] + ++[[package]] ++name = "bitflags" ++version = "1.3.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" ++ + [[package]] + name = "bitflags" + version = "2.11.0" +@@ -887,16 +908,15 @@ dependencies = [ + + [[package]] + name = "blake3" +-version = "1.8.3" ++version = "1.8.7" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" ++checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" + dependencies = [ +- "arrayref", + "arrayvec", + "cc", + "cfg-if 1.0.4", + "constant_time_eq", +- "cpufeatures 0.2.17", ++ "cpufeatures 0.3.0", + ] + + [[package]] +@@ -1127,6 +1147,12 @@ dependencies = [ + "cc", + ] + ++[[package]] ++name = "cmov" ++version = "0.5.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" ++ + [[package]] + name = "colorchoice" + version = "1.0.5" +@@ -1295,12 +1321,13 @@ dependencies = [ + ] + + [[package]] +-name = "crc32c" +-version = "0.6.8" ++name = "crc-fast" ++version = "1.10.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" ++checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" + dependencies = [ +- "rustc_version", ++ "digest 0.10.7", ++ "spin 0.10.1", + ] + + [[package]] +@@ -1437,6 +1464,15 @@ version = "0.0.7" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" + ++[[package]] ++name = "ctutils" ++version = "0.4.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" ++dependencies = [ ++ "cmov", ++] ++ + [[package]] + name = "darling" + version = "0.23.0" +@@ -1785,7 +1821,7 @@ checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" + dependencies = [ + "arrow", + "arrow-buffer", +- "base64", ++ "base64 0.22.1", + "blake2", + "blake3", + "chrono", +@@ -2112,6 +2148,37 @@ dependencies = [ + "url", + ] + ++[[package]] ++name = "defmt" ++version = "1.1.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" ++dependencies = [ ++ "bitflags 1.3.2", ++ "defmt-macros", ++] ++ ++[[package]] ++name = "defmt-macros" ++version = "1.1.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" ++dependencies = [ ++ "defmt-parser", ++ "proc-macro2", ++ "quote", ++ "syn 2.0.117", ++] ++ ++[[package]] ++name = "defmt-parser" ++version = "1.0.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" ++dependencies = [ ++ "thiserror 2.0.18", ++] ++ + [[package]] + name = "der" + version = "0.7.10" +@@ -2154,6 +2221,7 @@ dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", ++ "ctutils", + ] + + [[package]] +@@ -2322,7 +2390,7 @@ version = "25.12.19" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "rustc_version", + ] + +@@ -2369,6 +2437,12 @@ dependencies = [ + "percent-encoding", + ] + ++[[package]] ++name = "frostem" ++version = "1.20260821.5" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "36a80a7406da302e04bfd2ca987907590d3a1f3c69958947c43890abd7426b2f" ++ + [[package]] + name = "fs_extra" + version = "1.3.0" +@@ -2377,8 +2451,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + + [[package]] + name = "fsst" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "rand 0.9.2", +@@ -2727,14 +2801,23 @@ dependencies = [ + + [[package]] + name = "goosefs-sdk" +-version = "0.1.5" ++version = "0.1.9" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" ++checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" + dependencies = [ ++ "arc-swap", + "async-trait", + "bytes", + "dashmap", ++ "fastrand", ++ "futures", + "hostname", ++ "io-uring", ++ "itoa", ++ "libc", ++ "lru", ++ "memmap2", ++ "moka", + "prost", + "prost-types", + "rand 0.9.2", +@@ -2747,6 +2830,7 @@ dependencies = [ + "tonic-prost", + "tracing", + "uuid", ++ "xxhash-rust", + ] + + [[package]] +@@ -2900,6 +2984,15 @@ dependencies = [ + "digest 0.10.7", + ] + ++[[package]] ++name = "hmac" ++version = "0.13.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" ++dependencies = [ ++ "digest 0.11.3", ++] ++ + [[package]] + name = "hostname" + version = "0.4.2" +@@ -3053,7 +3146,7 @@ version = "0.1.20" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", +@@ -3347,7 +3440,7 @@ version = "0.7.12" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "cfg-if 1.0.4", + "libc", + ] +@@ -3400,10 +3493,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + + [[package]] + name = "jiff" +-version = "0.2.23" ++version = "0.2.35" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" ++checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" + dependencies = [ ++ "defmt", ++ "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", +@@ -3412,15 +3507,25 @@ dependencies = [ + "portable-atomic-util", + "serde_core", + "wasm-bindgen", +- "windows-sys 0.61.2", ++ "windows-link", ++] ++ ++[[package]] ++name = "jiff-core" ++version = "0.1.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" ++dependencies = [ ++ "defmt", + ] + + [[package]] + name = "jiff-static" +-version = "0.2.23" ++version = "0.2.35" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" ++checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" + dependencies = [ ++ "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.117", +@@ -3530,24 +3635,6 @@ dependencies = [ + "serde_json", + ] + +-[[package]] +-name = "jsonwebtoken" +-version = "10.4.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" +-dependencies = [ +- "aws-lc-rs", +- "base64", +- "getrandom 0.2.17", +- "js-sys", +- "pem", +- "serde", +- "serde_json", +- "signature", +- "simple_asn1", +- "zeroize", +-] +- + [[package]] + name = "konst" + version = "0.4.3" +@@ -3567,8 +3654,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + + [[package]] + name = "lance" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arc-swap", + "arrow", +@@ -3584,7 +3671,6 @@ dependencies = [ + "async-recursion", + "async-trait", + "async_cell", +- "aws-credential-types", + "byteorder", + "bytes", + "chrono", +@@ -3599,7 +3685,6 @@ dependencies = [ + "either", + "fst", + "futures", +- "half", + "humantime", + "itertools 0.14.0", + "lance-arrow", +@@ -3641,8 +3726,8 @@ dependencies = [ + + [[package]] + name = "lance-arrow" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", +@@ -3664,7 +3749,7 @@ dependencies = [ + [[package]] + name = "lance-arrow-scalar" + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", +@@ -3678,7 +3763,7 @@ dependencies = [ + [[package]] + name = "lance-arrow-stats" + version = "58.0.0" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-schema", +@@ -3687,8 +3772,8 @@ dependencies = [ + + [[package]] + name = "lance-bitpacking" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrayref", + "crunchy", +@@ -3728,20 +3813,19 @@ dependencies = [ + + [[package]] + name = "lance-core" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "async-trait", +- "byteorder", ++ "blake3", + "bytes", + "datafusion-common", + "datafusion-sql", + "futures", +- "itertools 0.14.0", + "lance-arrow", + "lance-derive", + "libc", +@@ -3752,13 +3836,13 @@ dependencies = [ + "object_store", + "pin-project", + "prost", ++ "quick_cache", + "rand 0.9.2", + "roaring", + "serde_json", + "snafu", + "tempfile", + "tokio", +- "tokio-stream", + "tokio-util", + "tracing", + "twox-hash", +@@ -3767,8 +3851,8 @@ dependencies = [ + + [[package]] + name = "lance-datafusion" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "arrow-array", +@@ -3788,7 +3872,6 @@ dependencies = [ + "jsonb", + "lance-arrow", + "lance-core", +- "lance-datagen", + "lance-geo", + "log", + "pin-project", +@@ -3800,8 +3883,8 @@ dependencies = [ + + [[package]] + name = "lance-datagen" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "arrow-array", +@@ -3818,8 +3901,8 @@ dependencies = [ + + [[package]] + name = "lance-derive" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "proc-macro2", + "quote", +@@ -3828,8 +3911,8 @@ dependencies = [ + + [[package]] + name = "lance-encoding" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-arith", + "arrow-array", +@@ -3854,8 +3937,6 @@ dependencies = [ + "num-traits", + "prost", + "prost-build", +- "rand 0.9.2", +- "strum", + "tokio", + "tracing", + "xxhash-rust", +@@ -3864,12 +3945,13 @@ dependencies = [ + + [[package]] + name = "lance-file" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", ++ "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", +@@ -3895,8 +3977,8 @@ dependencies = [ + + [[package]] + name = "lance-geo" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "datafusion", + "geo-traits", +@@ -3910,13 +3992,14 @@ dependencies = [ + + [[package]] + name = "lance-index" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arc-swap", + "arrow", + "arrow-arith", + "arrow-array", ++ "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", +@@ -3925,7 +4008,6 @@ dependencies = [ + "async-trait", + "bitvec", + "bytes", +- "chrono", + "crossbeam-queue", + "datafusion", + "datafusion-common", +@@ -3945,7 +4027,6 @@ dependencies = [ + "lance-bitpacking", + "lance-core", + "lance-datafusion", +- "lance-datagen", + "lance-encoding", + "lance-file", + "lance-geo", +@@ -3975,13 +4056,12 @@ dependencies = [ + "tempfile", + "tokio", + "tracing", +- "uuid", + ] + + [[package]] + name = "lance-index-core" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-schema", +@@ -4003,18 +4083,12 @@ dependencies = [ + + [[package]] + name = "lance-io" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", +- "arrow-arith", + "arrow-array", +- "arrow-buffer", +- "arrow-cast", +- "arrow-data", + "arrow-schema", +- "arrow-select", +- "async-recursion", + "async-trait", + "aws-config", + "aws-credential-types", +@@ -4022,10 +4096,8 @@ dependencies = [ + "bytes", + "chrono", + "futures", +- "goosefs-sdk", + "http 1.4.0", + "io-uring", +- "lance-arrow", + "lance-core", + "lance-namespace", + "log", +@@ -4037,34 +4109,37 @@ dependencies = [ + "pin-project", + "prost", + "rand 0.9.2", ++ "reqsign-core", ++ "reqsign-file-read-tokio", ++ "reqsign-google", + "serde", ++ "serde_json", + "tempfile", + "tokio", + "tracing", + "url", ++ "uuid", + ] + + [[package]] + name = "lance-linalg" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", +- "arrow-buffer", + "arrow-schema", + "cc", + "half", + "lance-arrow", + "lance-core", + "num-traits", +- "rand 0.9.2", + "rayon", + ] + + [[package]] + name = "lance-namespace" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "async-trait", +@@ -4076,9 +4151,9 @@ dependencies = [ + + [[package]] + name = "lance-namespace-reqwest-client" +-version = "0.8.6" ++version = "0.11.1" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" ++checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359" + dependencies = [ + "reqwest 0.12.28", + "serde", +@@ -4090,14 +4165,13 @@ dependencies = [ + + [[package]] + name = "lance-select" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "byteorder", +- "bytes", + "itertools 0.14.0", + "lance-core", + "roaring", +@@ -4106,8 +4180,8 @@ dependencies = [ + + [[package]] + name = "lance-table" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ + "arrow", + "arrow-array", +@@ -4115,6 +4189,7 @@ dependencies = [ + "arrow-ipc", + "arrow-schema", + "async-trait", ++ "blake3", + "byteorder", + "bytes", + "chrono", +@@ -4144,11 +4219,11 @@ dependencies = [ + + [[package]] + name = "lance-tokenizer" +-version = "9.1.0-beta.3" +-source = "git+https://github.com/lance-format/lance.git?rev=e934cc2c#e934cc2ceda2bd5f5aa37a953cc29f71d24bd5c0" ++version = "11.0.0" ++source = "git+https://github.com/lance-format/lance.git?rev=ab6b5bbe#ab6b5bbe46009ed78746b444df8db59a8bc5d842" + dependencies = [ ++ "frostem", + "icu_segmenter", +- "rust-stemmers", + "serde", + "stop-words", + "unicode-normalization", +@@ -4160,7 +4235,7 @@ version = "1.5.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + dependencies = [ +- "spin", ++ "spin 0.9.8", + ] + + [[package]] +@@ -4291,9 +4366,9 @@ dependencies = [ + + [[package]] + name = "log" +-version = "0.4.29" ++version = "0.4.34" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" ++checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + + [[package]] + name = "loom" +@@ -4308,6 +4383,15 @@ dependencies = [ + "tracing-subscriber", + ] + ++[[package]] ++name = "lru" ++version = "0.18.4" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" ++dependencies = [ ++ "hashbrown 0.17.1", ++] ++ + [[package]] + name = "lru-slab" + version = "0.1.2" +@@ -4399,6 +4483,15 @@ version = "2.8.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + ++[[package]] ++name = "memmap2" ++version = "0.9.11" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" ++dependencies = [ ++ "libc", ++] ++ + [[package]] + name = "mime" + version = "0.3.17" +@@ -4619,7 +4712,7 @@ version = "0.3.2" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + ] + + [[package]] +@@ -4648,7 +4741,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" + dependencies = [ + "async-trait", +- "base64", ++ "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", +@@ -4664,7 +4757,7 @@ dependencies = [ + "md-5 0.10.6", + "parking_lot", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.39.4", + "rand 0.10.1", + "reqwest 0.12.28", + "ring", +@@ -4683,9 +4776,9 @@ dependencies = [ + + [[package]] + name = "object_store_opendal" +-version = "0.57.0" ++version = "0.58.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" ++checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" + dependencies = [ + "async-trait", + "bytes", +@@ -4718,12 +4811,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + + [[package]] + name = "opendal" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" ++checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" + dependencies = [ + "ctor 1.0.13", + "opendal-core", ++ "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", +@@ -4741,24 +4835,22 @@ dependencies = [ + + [[package]] + name = "opendal-core" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" ++checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" + dependencies = [ + "anyhow", +- "base64", ++ "asyncband", ++ "base64 0.23.1", + "bytes", + "futures", + "http 1.4.0", +- "http-body 1.0.1", + "jiff", + "log", + "md-5 0.11.0", +- "mea", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", +- "reqwest 0.13.3", + "serde", + "serde_json", + "tokio", +@@ -4767,23 +4859,37 @@ dependencies = [ + "web-time", + ] + ++[[package]] ++name = "opendal-http-transport-reqwest" ++version = "0.58.2" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" ++dependencies = [ ++ "bytes", ++ "futures", ++ "http 1.4.0", ++ "http-body 1.0.1", ++ "opendal-core", ++ "reqwest 0.13.4", ++] ++ + [[package]] + name = "opendal-layer-concurrent-limit" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" ++checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" + dependencies = [ ++ "asyncband", + "futures", + "http 1.4.0", +- "mea", + "opendal-core", + ] + + [[package]] + name = "opendal-layer-logging" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" ++checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" + dependencies = [ + "log", + "opendal-core", +@@ -4791,9 +4897,9 @@ dependencies = [ + + [[package]] + name = "opendal-layer-retry" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" ++checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" + dependencies = [ + "backon", + "log", +@@ -4802,9 +4908,9 @@ dependencies = [ + + [[package]] + name = "opendal-layer-timeout" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" ++checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" + dependencies = [ + "opendal-core", + "tokio", +@@ -4812,17 +4918,17 @@ dependencies = [ + + [[package]] + name = "opendal-service-azblob" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" ++checksum = "6d278d2fb57661947d1c9fb44047e432b2782c48dc085b7abc99fe6e18c26cf8" + dependencies = [ +- "base64", ++ "base64 0.23.1", + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "opendal-service-azure-common", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -4833,17 +4939,18 @@ dependencies = [ + + [[package]] + name = "opendal-service-azdls" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" ++checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" + dependencies = [ +- "base64", ++ "asyncband", ++ "base64 0.23.1", + "bytes", + "http 1.4.0", + "log", + "opendal-core", + "opendal-service-azure-common", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -4853,9 +4960,9 @@ dependencies = [ + + [[package]] + name = "opendal-service-azure-common" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" ++checksum = "cfcc1bfdac4f54d9018c462dd32ad9e2f68fdf584f825c811550c8ffc87894cd" + dependencies = [ + "http 1.4.0", + "opendal-core", +@@ -4863,15 +4970,15 @@ dependencies = [ + + [[package]] + name = "opendal-service-cos" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" ++checksum = "bb021c128ebde42e6f3e719d4cfed27e994017aa503a7a1e5818bdb61fd67dc9" + dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-tencent-cos", +@@ -4880,9 +4987,9 @@ dependencies = [ + + [[package]] + name = "opendal-service-gcs" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" ++checksum = "da1f2a8c975fd22fea01f0409bcb8774f1ac02ad79863a3490098203121555d3" + dependencies = [ + "async-trait", + "bytes", +@@ -4890,7 +4997,7 @@ dependencies = [ + "log", + "opendal-core", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", +@@ -4901,9 +5008,9 @@ dependencies = [ + + [[package]] + name = "opendal-service-goosefs" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" ++checksum = "89fd71b80078f2983bd363e322fbebfa6a46d5fc56f17e4f23d76d5edb31226b" + dependencies = [ + "bytes", + "goosefs-sdk", +@@ -4915,9 +5022,9 @@ dependencies = [ + + [[package]] + name = "opendal-service-hf" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" ++checksum = "c8a3c8ec0c2918fa23f258fa28822f222ec8c1e3a501ded665b78bc65a9d7734" + dependencies = [ + "bytes", + "hf-xet", +@@ -4925,22 +5032,21 @@ dependencies = [ + "log", + "opendal-core", + "percent-encoding", +- "reqwest 0.13.3", + "serde", + "serde_json", + ] + + [[package]] + name = "opendal-service-oss" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" ++checksum = "8e3ce7a2ceb925e0f28f545b169eb57d04cec6116aae94b8584d2bdbe7d456c6" + dependencies = [ + "bytes", + "http 1.4.0", + "log", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -4949,18 +5055,18 @@ dependencies = [ + + [[package]] + name = "opendal-service-s3" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" ++checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" + dependencies = [ +- "base64", ++ "base64 0.23.1", + "bytes", +- "crc32c", ++ "crc-fast", + "http 1.4.0", + "log", + "md-5 0.11.0", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", +@@ -4970,14 +5076,14 @@ dependencies = [ + + [[package]] + name = "opendal-service-tos" +-version = "0.57.0" ++version = "0.58.2" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" ++checksum = "70c3c507c3a565b2feb4c7b5f653436acd34ca59ffc842a7671b5498b13b76bf" + dependencies = [ + "bytes", + "http 1.4.0", + "opendal-core", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-volcengine-tos", +@@ -5084,7 +5190,7 @@ version = "0.8.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "serde", + ] + +@@ -5122,16 +5228,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" + dependencies = [ + "digest 0.10.7", +- "hmac", ++ "hmac 0.12.1", + ] + + [[package]] + name = "pem" +-version = "3.0.6" ++version = "4.0.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" ++checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" + dependencies = [ +- "base64", ++ "base64 0.23.1", + "serde_core", + ] + +@@ -5392,6 +5498,28 @@ dependencies = [ + "serde", + ] + ++[[package]] ++name = "quick-xml" ++version = "0.41.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" ++dependencies = [ ++ "memchr", ++ "serde", ++] ++ ++[[package]] ++name = "quick_cache" ++version = "0.6.24" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" ++dependencies = [ ++ "ahash", ++ "equivalent", ++ "hashbrown 0.16.1", ++ "parking_lot", ++] ++ + [[package]] + name = "quinn" + version = "0.11.9" +@@ -5616,7 +5744,7 @@ version = "0.5.18" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + ] + + [[package]] +@@ -5697,9 +5825,9 @@ dependencies = [ + + [[package]] + name = "reqsign-aliyun-oss" +-version = "3.0.0" ++version = "3.1.4" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" ++checksum = "68d24d281f734a463093b7b93aae8b16f5f8496a54fbeec4d2d10b0488295296" + dependencies = [ + "anyhow", + "form_urlencoded", +@@ -5713,18 +5841,18 @@ dependencies = [ + ] + + [[package]] +-name = "reqsign-aws-v4" +-version = "3.0.0" ++name = "reqsign-aws-core" ++version = "3.1.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" ++checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" + dependencies = [ +- "anyhow", + "bytes", + "form_urlencoded", ++ "hex", + "http 1.4.0", + "log", + "percent-encoding", +- "quick-xml", ++ "quick-xml 0.41.0", + "reqsign-core", + "rust-ini", + "serde", +@@ -5733,18 +5861,32 @@ dependencies = [ + "sha1", + ] + ++[[package]] ++name = "reqsign-aws-v4" ++version = "3.2.0" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43" ++dependencies = [ ++ "bytes", ++ "http 1.4.0", ++ "log", ++ "quick-xml 0.41.0", ++ "reqsign-aws-core", ++ "reqsign-core", ++ "serde", ++] ++ + [[package]] + name = "reqsign-azure-storage" +-version = "3.0.0" ++version = "3.2.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" ++checksum = "e8177b4f08620ab7f2e9cab7d7ccb9da1b61c66b889fed46cc0880b0fe75eb6b" + dependencies = [ + "anyhow", +- "base64", ++ "base64 0.23.1", + "bytes", + "form_urlencoded", + "http 1.4.0", +- "jsonwebtoken", + "log", + "pem", + "percent-encoding", +@@ -5757,31 +5899,33 @@ dependencies = [ + + [[package]] + name = "reqsign-core" +-version = "3.0.0" ++version = "3.3.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" ++checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" + dependencies = [ + "anyhow", +- "base64", ++ "base64 0.23.1", + "bytes", +- "form_urlencoded", + "futures", + "hex", +- "hmac", ++ "hmac 0.13.0", + "http 1.4.0", + "jiff", + "log", + "percent-encoding", ++ "rsa", ++ "serde", ++ "serde_json", + "sha1", +- "sha2 0.10.9", ++ "sha2 0.11.0", + "windows-sys 0.61.2", + ] + + [[package]] + name = "reqsign-file-read-tokio" +-version = "3.0.0" ++version = "3.0.5" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" ++checksum = "95c3371bfc7e5c7f9627a04133af3583fd6c28715e7c83f79db38f3b384f535f" + dependencies = [ + "anyhow", + "reqsign-core", +@@ -5790,13 +5934,13 @@ dependencies = [ + + [[package]] + name = "reqsign-google" +-version = "3.0.0" ++version = "3.1.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" ++checksum = "f81a9d38870892443489c0abb5332edfa81d5a14c437af9caef7c194897c92c1" + dependencies = [ ++ "bytes", + "form_urlencoded", + "http 1.4.0", +- "jsonwebtoken", + "log", + "percent-encoding", + "reqsign-aws-v4", +@@ -5804,15 +5948,14 @@ dependencies = [ + "rsa", + "serde", + "serde_json", +- "sha2 0.10.9", + "tokio", + ] + + [[package]] + name = "reqsign-tencent-cos" +-version = "3.0.0" ++version = "3.0.5" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a" ++checksum = "b15c5a4df7c3f16823ae242675c5ebfb52d640cc9a50d1fcf943263247fa1730" + dependencies = [ + "anyhow", + "http 1.4.0", +@@ -5825,9 +5968,9 @@ dependencies = [ + + [[package]] + name = "reqsign-volcengine-tos" +-version = "3.0.0" ++version = "3.1.1" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "f9d757602a7ef2b6025c0da77e6d2e23fbdef35930fa466b15ffbf0a3f13acf7" ++checksum = "173387eb5ae4cf6a0a7098665ebcc6729862819dc3d95a81aee840767803e3d9" + dependencies = [ + "anyhow", + "http 1.4.0", +@@ -5842,7 +5985,7 @@ version = "0.12.28" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", +@@ -5884,11 +6027,11 @@ dependencies = [ + + [[package]] + name = "reqwest" +-version = "0.13.3" ++version = "0.13.4" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" ++checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", +@@ -5931,7 +6074,7 @@ dependencies = [ + "anyhow", + "async-trait", + "http 1.4.0", +- "reqwest 0.13.3", ++ "reqwest 0.13.4", + "thiserror 2.0.18", + "tower-service", + ] +@@ -5946,7 +6089,7 @@ dependencies = [ + "cfg-if 1.0.4", + "getrandom 0.2.17", + "libc", +- "untrusted 0.9.0", ++ "untrusted", + "windows-sys 0.52.0", + ] + +@@ -6008,16 +6151,6 @@ dependencies = [ + "ordered-multimap", + ] + +-[[package]] +-name = "rust-stemmers" +-version = "1.2.0" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +-dependencies = [ +- "serde", +- "serde_derive", +-] +- + [[package]] + name = "rustc-hash" + version = "2.1.1" +@@ -6039,7 +6172,7 @@ version = "1.1.4" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys", +@@ -6119,7 +6252,7 @@ dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", +- "untrusted 0.9.0", ++ "untrusted", + ] + + [[package]] +@@ -6244,7 +6377,7 @@ version = "3.7.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", +@@ -6373,7 +6506,7 @@ version = "3.20.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" + dependencies = [ +- "base64", ++ "base64 0.22.1", + "bs58", + "chrono", + "hex", +@@ -6414,13 +6547,13 @@ dependencies = [ + + [[package]] + name = "sha1" +-version = "0.10.6" ++version = "0.11.0" + source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" ++checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" + dependencies = [ + "cfg-if 1.0.4", +- "cpufeatures 0.2.17", +- "digest 0.10.7", ++ "cpufeatures 0.3.0", ++ "digest 0.11.3", + ] + + [[package]] +@@ -6523,18 +6656,6 @@ version = "0.1.5" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +-[[package]] +-name = "simple_asn1" +-version = "0.6.4" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +-dependencies = [ +- "num-bigint", +- "num-traits", +- "thiserror 2.0.18", +- "time", +-] +- + [[package]] + name = "siphasher" + version = "1.0.2" +@@ -6602,6 +6723,12 @@ version = "0.9.8" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + ++[[package]] ++name = "spin" ++version = "0.10.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" ++ + [[package]] + name = "spki" + version = "0.7.3" +@@ -6682,28 +6809,6 @@ version = "0.11.1" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +-[[package]] +-name = "strum" +-version = "0.26.3" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +-dependencies = [ +- "strum_macros", +-] +- +-[[package]] +-name = "strum_macros" +-version = "0.26.4" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +-dependencies = [ +- "heck", +- "proc-macro2", +- "quote", +- "rustversion", +- "syn 2.0.117", +-] +- + [[package]] + name = "substrait" + version = "0.63.0" +@@ -6804,7 +6909,7 @@ version = "0.7.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "core-foundation 0.9.4", + "system-configuration-sys", + ] +@@ -7078,7 +7183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" + dependencies = [ + "async-trait", +- "base64", ++ "base64 0.22.1", + "bytes", + "h2", + "http 1.4.0", +@@ -7136,7 +7241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" + dependencies = [ + "async-compression", +- "bitflags", ++ "bitflags 2.11.0", + "bytes", + "futures-core", + "futures-util", +@@ -7370,12 +7475,6 @@ version = "0.2.11" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +-[[package]] +-name = "untrusted" +-version = "0.7.1" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +- + [[package]] + name = "untrusted" + version = "0.9.0" +@@ -7622,7 +7721,7 @@ version = "0.244.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" + dependencies = [ +- "bitflags", ++ "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "semver", +@@ -8054,7 +8153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" + dependencies = [ + "anyhow", +- "bitflags", ++ "bitflags 2.11.0", + "indexmap 2.14.0", + "log", + "serde", +@@ -8132,7 +8231,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" + dependencies = [ + "anyhow", + "async-trait", +- "base64", ++ "base64 0.22.1", + "bytes", + "clap", + "crc32fast", +@@ -8143,7 +8242,7 @@ dependencies = [ + "more-asserts", + "rand 0.10.1", + "redb", +- "reqwest 0.13.3", ++ "reqwest 0.13.4", + "reqwest-middleware", + "serde", + "serde_json", +@@ -8169,7 +8268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" + dependencies = [ + "async-trait", +- "base64", ++ "base64 0.22.1", + "blake3", + "bytemuck", + "bytes", +@@ -8256,7 +8355,7 @@ dependencies = [ + "oneshot", + "pin-project", + "rand 0.10.1", +- "reqwest 0.13.3", ++ "reqwest 0.13.4", + "serde", + "serde_json", + "shellexpand", +@@ -8352,20 +8451,6 @@ name = "zeroize" + version = "1.8.2" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +-dependencies = [ +- "zeroize_derive", +-] +- +-[[package]] +-name = "zeroize_derive" +-version = "1.4.3" +-source = "registry+https://github.com/rust-lang/crates.io-index" +-checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +-dependencies = [ +- "proc-macro2", +- "quote", +- "syn 2.0.117", +-] + + [[package]] + name = "zerotrie" +diff --git a/Cargo.toml b/Cargo.toml +index d072a5d..3920a65 100644 +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -18,14 +18,14 @@ rust-version = "1.91.0" + crate-type = ["cdylib", "staticlib", "rlib"] + + [dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] } +-lance-core = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-index = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-io = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-table = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } ++lance-core = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-index = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-io = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-table = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-datafusion = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } + datafusion = { version = "54.0.0", default-features = false } + arrow = { version = "58.0.0", features = ["prettyprint", "ffi"] } + arrow-array = "58.0.0" +@@ -45,9 +45,9 @@ snafu = "0.9" + uuid = { version = "1", features = ["v4"] } + + [dev-dependencies] +-lance = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c", features = ["substrait"] } +-lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } +-lance-file = { git = "https://github.com/lance-format/lance.git", rev = "e934cc2c" } ++lance = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe", features = ["substrait"] } ++lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } ++lance-file = { git = "https://github.com/lance-format/lance.git", rev = "ab6b5bbe" } + tokio = { version = "1", features = ["rt-multi-thread", "macros"] } + arrow-array = "58.0.0" + arrow-schema = "58.0.0" +diff --git a/src/fts_query.rs b/src/fts_query.rs +index c9d3a43..71874e0 100644 +--- a/src/fts_query.rs ++++ b/src/fts_query.rs +@@ -246,7 +246,7 @@ async fn prepare_fts_query_context( + .with_max_expansions(match_query.max_expansions) + .with_prefix_length(match_query.prefix_length); + PreparedFtsQuery::Match(Arc::new( +- build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?, ++ build_global_bm25_scorer(&indices, &query_tokens, ¶ms, None).await?, + )) + } + FtsQuery::Phrase(phrase_query) => { +@@ -260,7 +260,7 @@ async fn prepare_fts_query_context( + let query_tokens = collect_query_tokens(&phrase_query.terms, &mut tokenizer); + let params = query.params().with_phrase_slop(Some(phrase_query.slop)); + PreparedFtsQuery::Phrase(Arc::new( +- build_global_bm25_scorer(&indices, &query_tokens, ¶ms).await?, ++ build_global_bm25_scorer(&indices, &query_tokens, ¶ms, None).await?, + )) + } + _ => { +diff --git a/src/index_segment.rs b/src/index_segment.rs +index a46c4f3..d4e143c 100644 +--- a/src/index_segment.rs ++++ b/src/index_segment.rs +@@ -889,7 +889,7 @@ unsafe fn new_vector_builder_inner( + // TODO(upstream-lance): Remove this fail-fast once Lance's distributed + // vector-index path reconstructs a supplied PQ codebook with an L2 + // ProductQuantizer, matching the ordinary full-dataset path. Pinned Lance +- // revision e934cc2c rewraps supplied codebooks with DistanceType::Dot in ++ // revision ab6b5bbe rewraps supplied codebooks with DistanceType::Dot in + // `make_global_pq`, which silently switches PQ code assignment away from + // the L2 contract shared by full-dataset builds and index readers. + if matches!( +@@ -910,7 +910,7 @@ unsafe fn new_vector_builder_inner( + let selected_fragment_ids: HashSet = fragment_ids.iter().copied().collect(); + if selected_fragment_ids != all_fragment_ids { + return Err(invalid_input(format!( +- "pq_codebook is supplied for metric=DOT, index_type={:?}, mode={:?}, and an effective strict fragment subset ({} of {} fragments): pinned Lance revision e934cc2c reconstructs the supplied codebook with a DOT ProductQuantizer in the distributed build path (make_global_pq), silently breaking the L2 PQ-assignment contract; cover the full dataset in one segment (pass NULL fragment_ids or list every fragment) or wait for upstream Lance DOT support", ++ "pq_codebook is supplied for metric=DOT, index_type={:?}, mode={:?}, and an effective strict fragment subset ({} of {} fragments): pinned Lance revision ab6b5bbe reconstructs the supplied codebook with a DOT ProductQuantizer in the distributed build path (make_global_pq), silently breaking the L2 PQ-assignment contract; cover the full dataset in one segment (pass NULL fragment_ids or list every fragment) or wait for upstream Lance DOT support", + params.index_type, + parsed.mode, + selected_fragment_ids.len(), +diff --git a/src/scanner.rs b/src/scanner.rs +index 7e898ce..d3ef3be 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -529,7 +529,7 @@ fn rewrite_prepared_fts_plan( + exec.params().clone(), + exec.prefilter_source().clone(), + segments.to_vec(), +- ) ++ )? + .with_base_scorer(Arc::clone(scorer)); + return Ok((Arc::new(replacement), rewritten)); + } +@@ -549,7 +549,7 @@ fn rewrite_prepared_fts_plan( + exec.params().clone(), + exec.prefilter_source().clone(), + segments.to_vec(), +- ) ++ )? + .with_base_scorer(Arc::clone(scorer)); + return Ok((Arc::new(replacement), rewritten)); + } +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index b763cef..bde742d 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -2449,8 +2449,7 @@ fn test_robotics_e2e_write_then_finalize() { + format!("data/{}", filename), + field_ids, + column_indices, +- meta.major_version as u32, +- meta.minor_version as u32, ++ meta.version, + None, // file_size_bytes + None, // base_id + ); +@@ -3405,6 +3404,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() { + created_at: Some(u64::MAX), + base_id: None, + files: Vec::new(), ++ covering_fields: Vec::new(), + } + .encode_to_vec(); + assert_eq!( +@@ -3429,6 +3429,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() { + created_at: None, + base_id: None, + files: Vec::new(), ++ covering_fields: Vec::new(), + } + .encode_to_vec(); + assert_eq!( +@@ -3456,6 +3457,7 @@ fn test_index_segment_metadata_parse_rejects_malformed_and_dangerous_input() { + created_at: None, + base_id: None, + files: Vec::new(), ++ covering_fields: Vec::new(), + } + .encode_to_vec(); + assert_eq!( +@@ -4033,7 +4035,7 @@ fn test_vector_index_segment_rejects_strict_subset_dot_pq() { + let message = take_last_error_message(); + assert!(message.contains("metric=DOT"), "{message}"); + assert!(message.contains("strict fragment subset"), "{message}"); +- assert!(message.contains("e934cc2c"), "{message}"); ++ assert!(message.contains("ab6b5bbe"), "{message}"); + assert!(message.contains("1 of 2 fragments"), "{message}"); + assert!(!centroids.is_released()); + assert!(!codebook.is_released()); diff --git a/thirdparty/patches/lance-c-0.1.9-pr-79.patch b/thirdparty/patches/lance-c-0.1.9-pr-79.patch new file mode 100644 index 00000000000000..ad32a48065f09d --- /dev/null +++ b/thirdparty/patches/lance-c-0.1.9-pr-79.patch @@ -0,0 +1,1782 @@ +From d819fbdfa52031d84d1fa01f2d06c51a6712c40b Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Tue, 8 Sep 2026 22:30:48 +0800 +Subject: [PATCH 1/5] scalar index segment + +--- + docs/scalar-segment-scans.md | 78 +++++++++++ + include/lance/lance.h | 24 ++++ + include/lance/lance.hpp | 14 ++ + src/lib.rs | 1 + + src/scalar_segment.rs | 224 +++++++++++++++++++++++++++++++ + src/scanner.rs | 61 +++++++++ + tests/c_api_test.rs | 249 +++++++++++++++++++++++++++++++++++ + 7 files changed, 651 insertions(+) + create mode 100644 docs/scalar-segment-scans.md + create mode 100644 src/scalar_segment.rs + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +new file mode 100644 +index 0000000..dc7c141 +--- /dev/null ++++ b/docs/scalar-segment-scans.md +@@ -0,0 +1,78 @@ ++# Scalar index segment scans ++ ++An ordinary scanner can use one physical BTree/Bitmap segment to generate ++candidates, then read those candidates with the complete scanner filter. This ++does not run a global search of the other segments of the logical index. It does ++not subdivide a physical segment or make its own index search incremental. ++ ++## Configuring a task ++ ++Open a fixed dataset version. Select the physical index UUID from that version's ++metadata and pass the task's complete fragment domain explicitly: ++ ++```c ++LanceScanner *scanner = lance_scanner_new(dataset, columns, full_filter_sql); ++/* Check every return value in production. */ ++lance_scanner_set_fragment_ids(scanner, fragment_ids, fragment_count); ++lance_scanner_set_scalar_index_segment(scanner, segment_uuid_16_bytes); ++lance_scanner_set_limit(scanner, 20000); ++/* The scanner-owning thread calls lance_scanner_next as usual. */ ++``` ++ ++SQL, Substrait and additional SQL filters keep their existing precedence and AND ++composition. The caller does not supply a separate driver predicate: Lance-C ++uses the typed filter planner and selects a necessary indexed leaf belonging to ++the requested logical index. It only descends through AND, never through OR or ++NOT. It then searches the selected UUID and applies the complete filter while ++reading candidates with automatic scalar-index planning disabled. ++ ++Each task's fragment IDs define its result domain, including on fallback. A ++distributed planner must assign disjoint domains whose union covers the intended ++scan. Unindexed fragments need their own tasks, or an explicit domain including ++them (which causes that task to use fallback). Merely listing indexed segments ++does not include appended, unindexed data automatically. ++ ++An unknown UUID, absent fragment or invalid option combination is an error. A ++known segment with incomplete/unknown coverage, no suitable driver, unsupported ++index type, nested key, overlays, fragment reuse, non-exact results or unsupported ++row-ID domain falls back to a non-indexed scan of the entire explicit domain. ++I/O and corruption errors are propagated, not converted to empty results or ++successful fallback. ++ ++The first implementation supports live-row ordinary scans and cannot be combined ++with vector/FTS queries. Physical row-address ++results on stable-row-ID datasets currently fall back; results already expressed ++in the correct row-ID domain use the candidate path. Deletes and all remaining ++predicates are handled by the ordinary reader. No candidate-count limit is ++applied: LIMIT/OFFSET remain after the scanner's complete filter. ++ ++If the host has additional predicates outside Lance, do not set a local limit ++before those predicates. Never divide the global limit by the number of tasks. ++Global OFFSET belongs to the coordinator, not independently to each task. ++ ++## Stopping after the host limit ++ ++This mode uses the existing scanner/stream lifecycle. Once the host has enough ++rows, it stops requesting further batches and closes the scanner after any active ++call has returned. Do not call `lance_scanner_close` concurrently with `next`. ++Exported Arrow streams remain owned by the caller and must also be released after ++their active consumers have finished. ++ ++A host stop flag does not interrupt an in-progress `lance_scanner_next`: current ++index evaluation or I/O may finish before the host observes stop and closes the ++stream. No separate cancellation signal or thread is introduced. The host remains ++responsible for enforcing the global LIMIT across concurrent tasks. ++ ++## Memory and statistics ++ ++Candidate masks stay in Rust, and record batches are streamed. Each active task ++can still hold a complete segment's candidate set; scanner I/O buffer size does ++not cap that allocation. Control task concurrency and physical segment size. ++ ++Successful exhaustion merges segment-search metrics into the existing statistics ++callback exactly once. New metrics include `scalar_segments_requested`, ++`scalar_segments_searched`, `scalar_segment_candidate_rows`, ++`scalar_segment_prepare_time`, `scalar_segment_search_time`, and ++`scalar_segment_fallback_*` reasons. `prepare_time` includes search time. Early ++release, cancellation and errors retain the existing callback contract: final ++statistics are not guaranteed. Metrics do not establish global task concurrency. +diff --git a/include/lance/lance.h b/include/lance/lance.h +index 8173ae5..cbd8330 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1858,6 +1858,30 @@ int32_t lance_scanner_set_index_segments( + size_t len + ); + ++/** ++ * Accelerate an ordinary scalar-filtered scan with one physical index segment. ++ * segment_uuid points to 16 UUID bytes in RFC 4122 order; NULL clears the setting. ++ * Must be configured before scanning. Requires explicit nonempty fragment_ids, ++ * which define BOTH the read and fallback domain, independently of the segment. ++ * Missing snapshot UUIDs / fragment IDs are errors. Extra segment coverage is ++ * excluded by fragment_ids; incomplete coverage falls back to a full filtered ++ * scan of those fragment_ids. Callers distributing work must assign disjoint ++ * fragment domains and separately include any unindexed data they wish to read. ++ * ++ * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter ++ * on the selected logical index. All predicates are reapplied during candidate ++ * reads; other scalar indices are disabled. OR/NOT-only filters, overlays, ++ * fragment reuse, unsupported index types / result domains ++ * and missing coverage use the same domain without an index. No filter also ++ * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the ++ * unfiltered candidate set. Vector/FTS queries are rejected. ++ * ++ * UUID bytes are copied. Metadata and final option compatibility are validated ++ * when creating the stream. Index corruption or I/O failures remain errors. ++ */ ++int32_t lance_scanner_set_scalar_index_segment( ++ LanceScanner* scanner, const uint8_t* segment_uuid); ++ + /* ─── Full-text search (Phase 2) ─── */ + + /** +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index c12c0c6..8e1c6cf 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1311,6 +1311,20 @@ class Scanner { + return *this; + } + ++ /// Restrict scalar candidate generation to one segment; fragment_ids is ++ /// required and defines the complete read/fallback domain. See lance.h. ++ Scanner& scalar_index_segment(const std::array& segment_uuid) { ++ if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0) ++ check_error(); ++ return *this; ++ } ++ ++ Scanner& clear_scalar_index_segment() { ++ if (lance_scanner_set_scalar_index_segment(handle_.get(), nullptr) != 0) ++ check_error(); ++ return *this; ++ } ++ + /// Restrict scan to specific fragment IDs. + Scanner& fragment_ids(const uint64_t* ids, size_t len) { + if (lance_scanner_set_fragment_ids(handle_.get(), ids, len) != 0) +diff --git a/src/lib.rs b/src/lib.rs +index 8b212f5..c7ca4cf 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -39,6 +39,7 @@ mod index_segment; + mod merge_insert; + mod restore; + pub mod runtime; ++mod scalar_segment; + mod scanner; + mod session; + pub mod stream_guard; +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +new file mode 100644 +index 0000000..2ab1297 +--- /dev/null ++++ b/src/scalar_segment.rs +@@ -0,0 +1,224 @@ ++// SPDX-License-Identifier: Apache-2.0 ++// SPDX-FileCopyrightText: Copyright The Lance Authors ++ ++//! Segment-scoped candidate generation for ordinary scans. The explicit fragment ++//! list is the read domain, including on fallback; a segment is only an accelerator. ++ ++use std::collections::HashSet; ++use std::sync::Arc; ++use std::time::Instant; ++ ++use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; ++use lance::Dataset; ++use lance::dataset::scanner::{ ++ ExecutionStatsCallback, ExecutionSummaryCounts, RowAddrMask, Scanner, ++}; ++use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; ++use lance::io::exec::utils::IndexMetrics; ++use lance_core::{Error, Result}; ++use lance_datafusion::planner::Planner; ++use lance_datafusion::utils::MetricsExt; ++use lance_index::IndexType; ++use lance_index::scalar::SearchResult; ++use lance_index::scalar::expression::{PlannerIndexExt, ScalarIndexExpr, ScalarIndexSearch}; ++use uuid::Uuid; ++ ++pub(crate) struct PreparedScalarSegment { ++ pub dataset: Arc, ++ pub segment_uuid: Uuid, ++ pub fragment_ids: Vec, ++ pub callback: Option, ++} ++ ++fn invalid(message: impl Into) -> Error { ++ Error::invalid_input_source(message.into().into()) ++} ++ ++// Only descend through AND: a leaf below OR or NOT need not contain all matches ++// of the full expression. The original expression is always reapplied by reader. ++fn driver<'a>(expr: &'a ScalarIndexExpr, index_name: &str) -> Option<&'a ScalarIndexSearch> { ++ match expr { ++ ScalarIndexExpr::Query(search) if search.index_name == index_name => Some(search), ++ ScalarIndexExpr::And(lhs, rhs) => { ++ driver(lhs, index_name).or_else(|| driver(rhs, index_name)) ++ } ++ _ => None, ++ } ++} ++ ++impl PreparedScalarSegment { ++ pub async fn configure(self, mut reader: Scanner) -> Result { ++ // Never let either candidate reads or fallback re-enter a global index search. ++ reader.use_scalar_index(false); ++ let mut stats = ExecutionSummaryCounts::default(); ++ stats ++ .all_counts ++ .insert("scalar_segments_requested".into(), 1); ++ let plan_metrics = ExecutionPlanMetricsSet::new(); ++ let metrics = IndexMetrics::new(&plan_metrics, 0); ++ let started = Instant::now(); ++ let reason = self ++ .configure_candidates(&mut reader, &metrics, &mut stats) ++ .await?; ++ metrics.flush_io(); ++ stats.all_times.insert( ++ "scalar_segment_prepare_time".into(), ++ started.elapsed().as_nanos().min(usize::MAX as u128) as usize, ++ ); ++ if let Some(reason) = reason { ++ stats ++ .all_counts ++ .insert("scalar_segment_fallbacks".into(), 1); ++ stats ++ .all_counts ++ .insert(format!("scalar_segment_fallback_{reason}"), 1); ++ } ++ for (name, count) in plan_metrics.clone_inner().iter_counts() { ++ let name = name.as_ref(); ++ match name { ++ "iops" => stats.iops += count.value(), ++ "requests" => stats.requests += count.value(), ++ "bytes_read" => stats.bytes_read += count.value(), ++ "indices_loaded" => stats.indices_loaded += count.value(), ++ "parts_loaded" => stats.parts_loaded += count.value(), ++ "index_comparisons" => stats.index_comparisons += count.value(), ++ _ => *stats.all_counts.entry(name.to_string()).or_default() += count.value(), ++ } ++ } ++ if let Some(callback) = self.callback { ++ // Preserve the callback's once-per-successfully-exhausted-stream contract. ++ // Candidate work is not part of the underlying reader's plan metrics. ++ reader.scan_stats_callback(Arc::new(move |read| { ++ let mut combined = read.clone(); ++ combined.iops += stats.iops; ++ combined.requests += stats.requests; ++ combined.bytes_read += stats.bytes_read; ++ combined.indices_loaded += stats.indices_loaded; ++ combined.parts_loaded += stats.parts_loaded; ++ combined.index_comparisons += stats.index_comparisons; ++ for (name, value) in &stats.all_counts { ++ *combined.all_counts.entry(name.clone()).or_default() += value; ++ } ++ for (name, value) in &stats.all_times { ++ *combined.all_times.entry(name.clone()).or_default() += value; ++ } ++ callback(&combined); ++ })); ++ } ++ Ok(reader) ++ } ++ ++ async fn configure_candidates( ++ &self, ++ reader: &mut Scanner, ++ metrics: &IndexMetrics, ++ stats: &mut ExecutionSummaryCounts, ++ ) -> Result> { ++ let fragments = self.dataset.get_fragments(); ++ let visible: HashSet = fragments.iter().map(|f| f.id() as u64).collect(); ++ if self.fragment_ids.iter().any(|id| !visible.contains(id)) { ++ return Err(invalid( ++ "scalar segment fragment_ids contains a fragment absent from the dataset snapshot", ++ )); ++ } ++ let indices = self.dataset.load_indices().await?; ++ let index_meta = indices ++ .iter() ++ .find(|i| i.uuid == self.segment_uuid) ++ .ok_or_else(|| { ++ invalid(format!( ++ "scalar index segment {} is absent from the dataset snapshot", ++ self.segment_uuid ++ )) ++ })?; ++ let field_id = index_meta ++ .keyed_field() ++ .ok_or_else(|| invalid("scalar segment must index a single key field"))?; ++ let field = ++ self.dataset.schema().field_by_id(field_id).ok_or_else(|| { ++ invalid("scalar segment key field is absent from the dataset schema") ++ })?; ++ // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove ++ // the field path of an evolved or nested schema. ++ if !self ++ .dataset ++ .schema() ++ .fields ++ .iter() ++ .any(|f| f.id == field.id) ++ { ++ return Ok(Some("nested_field")); ++ } ++ let scope: HashSet = self.fragment_ids.iter().copied().collect(); ++ let Some(coverage) = index_meta.fragment_bitmap.as_ref() else { ++ return Ok(Some("unknown_coverage")); ++ }; ++ if self ++ .fragment_ids ++ .iter() ++ .any(|id| u32::try_from(*id).map_or(true, |id| !coverage.contains(id))) ++ { ++ // Scan the ENTIRE explicit read domain, not just the covered part. ++ return Ok(Some("partial_coverage")); ++ } ++ if fragments ++ .iter() ++ .filter(|f| scope.contains(&(f.id() as u64))) ++ .any(|f| !f.metadata().overlays.is_empty() || f.metadata().physical_rows.is_none()) ++ { ++ return Ok(Some("fragment_state")); ++ } ++ // Fragment reuse can change the domain of an old segment. Until its ++ // coverage mapping is handled here, preserve correctness with a scoped scan. ++ if self.dataset.frag_reuse_index_uuid().await.is_some() { ++ return Ok(Some("fragment_reuse")); ++ } ++ let Some(filter) = reader.get_expr_filter()? else { ++ return Ok(Some("no_filter")); ++ }; ++ let planner = Planner::new(Arc::new(self.dataset.schema().into())); ++ let index_info = self.dataset.scalar_index_info().await?; ++ let filter_plan = planner.create_filter_plan(filter, &index_info, true)?; ++ let Some(search) = filter_plan ++ .index_query ++ .as_ref() ++ .and_then(|expr| driver(expr, &index_meta.name)) ++ else { ++ return Ok(Some("no_driver")); ++ }; ++ if search.column != field.name { ++ return Ok(Some("field_path")); ++ } ++ let index = self ++ .dataset ++ .open_scalar_index(&search.column, &self.segment_uuid, metrics) ++ .await?; ++ if !matches!(index.index_type(), IndexType::BTree | IndexType::Bitmap) { ++ return Ok(Some("index_type")); ++ } ++ // External masks use _rowid, not necessarily physical row addresses. ++ if index.results_are_row_addresses() && self.dataset.manifest.uses_stable_row_ids() { ++ return Ok(Some("row_id_domain")); ++ } ++ let started = Instant::now(); ++ let result = index.search(search.query.as_ref(), metrics).await?; ++ stats.all_times.insert( ++ "scalar_segment_search_time".into(), ++ started.elapsed().as_nanos().min(usize::MAX as u128) as usize, ++ ); ++ stats ++ .all_counts ++ .insert("scalar_segments_searched".into(), 1); ++ let SearchResult::Exact(rows) = result else { ++ return Ok(Some("inexact_result")); ++ }; ++ stats.all_counts.insert( ++ "scalar_segment_candidate_rows".into(), ++ rows.len().unwrap_or(0) as usize, ++ ); ++ // Do not truncate candidates at LIMIT. The reader evaluates the complete ++ // filter before applying its existing limit/offset operators. ++ reader.with_row_addr_prefilter(RowAddrMask::from_allowed(rows.selected_rows().clone())); ++ Ok(None) ++ } ++} +diff --git a/src/scanner.rs b/src/scanner.rs +index 4ceeb0e..ac3cff6 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -39,6 +39,7 @@ use crate::fts_query::{ + }; + use crate::helpers; + use crate::runtime::{RT, block_on}; ++use crate::scalar_segment::PreparedScalarSegment; + use crate::stream_guard::GuardedReader; + + /// Data type tag for query vectors, mirroring the C enum `LanceDataType`. +@@ -108,6 +109,7 @@ pub struct LanceScanner { + include_deleted_rows: bool, + fragment_ids: Option>, + index_segments: Option>, ++ scalar_index_segment: Option, + nearest: Option, + nprobes: NprobesRange, + approx_mode: Option, +@@ -256,6 +258,7 @@ impl LanceScanner { + include_deleted_rows: false, + fragment_ids: None, + index_segments: None, ++ scalar_index_segment: None, + nearest: None, + nprobes: NprobesRange::default(), + approx_mode: None, +@@ -470,12 +473,37 @@ impl LanceScanner { + None + }; + self.apply_filter(&mut scanner)?; ++ let scalar_segment = if let Some(segment_uuid) = self.scalar_index_segment { ++ if self.nearest.is_some() ++ || self.fts_query.is_some() ++ || self.fts_context.is_some() ++ || self.index_segments.is_some() ++ || self.fts_index_segments.is_some() ++ { ++ return Err(lance_core::Error::invalid_input_source( ++ "scalar_index_segment requires an ordinary scan of live rows".into(), ++ )); ++ } ++ let fragment_ids = self.fragment_ids.as_ref().filter(|ids| !ids.is_empty()) ++ .ok_or_else(|| lance_core::Error::invalid_input_source( ++ "scalar_index_segment requires explicit nonempty fragment_ids for its read and fallback domain".into(), ++ ))?; ++ Some(PreparedScalarSegment { ++ dataset: Arc::clone(&self.dataset), ++ segment_uuid, ++ fragment_ids: fragment_ids.clone(), ++ callback: self.scan_statistics_callback.clone(), ++ }) ++ } else { ++ None ++ }; + if let Some(callback) = &self.scan_statistics_callback { + scanner.scan_stats_callback(callback.clone()); + } + Ok(PreparedScanner { + scanner, + distributed_fts, ++ scalar_segment, + }) + } + } +@@ -490,10 +518,18 @@ struct PreparedFtsExecution { + struct PreparedScanner { + scanner: lance::dataset::scanner::Scanner, + distributed_fts: Option, ++ scalar_segment: Option, + } + + impl PreparedScanner { + async fn try_into_stream(self) -> Result { ++ if let Some(scalar_segment) = self.scalar_segment { ++ return scalar_segment ++ .configure(self.scanner) ++ .await? ++ .try_into_stream() ++ .await; ++ } + let Some(distributed_fts) = self.distributed_fts else { + return self.scanner.try_into_stream().await; + }; +@@ -858,6 +894,31 @@ macro_rules! scanner_ffi_try { + }}; + } + ++/// Select one physical scalar index segment. NULL clears the selection. ++/// Requires explicit fragment_ids and an ordinary live-row scan. See the C header. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_scalar_index_segment( ++ scanner: *mut LanceScanner, ++ segment_uuid: *const u8, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, { ++ let scanner = unsafe { scanner.as_mut() } ++ .ok_or_else(|| lance_core::Error::invalid_input_source("scanner is NULL".into()))?; ++ scanner.ensure_scan_not_started("scalar_index_segment")?; ++ let segment = if segment_uuid.is_null() { ++ None ++ } else { ++ Some( ++ Uuid::from_slice(unsafe { std::slice::from_raw_parts(segment_uuid, 16) }) ++ .map_err(|e| lance_core::Error::invalid_input_source(e.into()))?, ++ ) ++ }; ++ scanner.scalar_index_segment = segment; ++ Ok(0) ++ }) ++} ++ + // --------------------------------------------------------------------------- + // Scanner lifecycle + builder + // --------------------------------------------------------------------------- +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 3b3424b..a8a7eab 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12513,3 +12513,252 @@ fn test_add_columns_stream_null_dataset_consumes_stream() { + assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); + assert_stream_consumed(&stream, &drop_count); + } ++ ++// Segment scans deliberately use an unprojected nullable key and a residual ++// predicate so a candidate LIMIT or loss of filter columns changes the answer. ++fn create_scalar_segment_fixture( ++ kind: lance_index::IndexType, ++ stable: bool, ++) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { ++ use lance::dataset::WriteParams; ++ use lance::index::DatasetIndexExt; ++ use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; ++ let tmp = tempfile::tempdir().unwrap(); ++ let uri = tmp.path().join("segments").to_str().unwrap().to_owned(); ++ let uuids = lance_c::runtime::block_on(async { ++ let schema = Arc::new(Schema::new(vec![ ++ Field::new("id", DataType::Int32, false), ++ Field::new("key", DataType::Int32, true), ++ ])); ++ let batch = RecordBatch::try_new( ++ schema.clone(), ++ vec![ ++ Arc::new(Int32Array::from_iter_values(0..12)), ++ Arc::new(Int32Array::from( ++ (0..12) ++ .map(|id| if id % 4 == 0 { None } else { Some(id % 3) }) ++ .collect::>(), ++ )), ++ ], ++ ) ++ .unwrap(); ++ let mut ds = Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), ++ &uri, ++ Some(WriteParams { ++ max_rows_per_file: 4, ++ enable_stable_row_ids: stable, ++ ..Default::default() ++ }), ++ ) ++ .await ++ .unwrap(); ++ let params = ScalarIndexParams::for_builtin(if kind == lance_index::IndexType::Bitmap { ++ BuiltinIndexType::Bitmap ++ } else { ++ BuiltinIndexType::BTree ++ }); ++ let fragments = ds.get_fragments(); ++ assert_eq!(fragments.len(), 3); ++ let mut segments = Vec::new(); ++ for fragment in fragments.iter().take(2) { ++ segments.push( ++ ds.create_index_builder(&["key"], kind, ¶ms) ++ .name("key_idx".into()) ++ .fragments(vec![fragment.id() as u32]) ++ .execute_uncommitted() ++ .await ++ .unwrap(), ++ ); ++ } ++ let uuids = segments.iter().map(|s| *s.uuid.as_bytes()).collect(); ++ ds.commit_existing_index_segments("key_idx", "key", segments) ++ .await ++ .unwrap(); ++ uuids ++ }); ++ (tmp, uri, uuids) ++} ++ ++fn scalar_segment_ids( ++ uri: &str, ++ uuid: &[u8; 16], ++ fragments: &[u64], ++ filter: &str, ++ limit: Option, ++ offset: i64, ++) -> (Vec, CapturedScanStatistics) { ++ let uri = c_str(uri); ++ let filter = c_str(filter); ++ let id = c_str("id"); ++ let columns = [id.as_ptr(), ptr::null()]; ++ let mut captured = CapturedScanStatistics::default(); ++ let mut ids = Vec::new(); ++ unsafe { ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ assert!(!ds.is_null()); ++ let scanner = lance_scanner_new(ds, columns.as_ptr(), filter.as_ptr()); ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, fragments.as_ptr(), fragments.len()), ++ 0 ++ ); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), ++ 0 ++ ); ++ if let Some(limit) = limit { ++ assert_eq!(lance_scanner_set_limit(scanner, limit), 0); ++ } ++ assert_eq!(lance_scanner_set_offset(scanner, offset), 0); ++ assert_eq!( ++ lance_scanner_set_statistics_callback( ++ scanner, ++ Some(capture_scan_statistics), ++ (&mut captured as *mut CapturedScanStatistics).cast() ++ ), ++ 0 ++ ); ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ let rc = lance_scanner_to_arrow_stream(scanner, &mut stream); ++ assert_eq!( ++ rc, ++ 0, ++ "{}", ++ if rc != 0 { ++ take_last_error_message() ++ } else { ++ String::new() ++ } ++ ); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, ptr::null()), ++ -1 ++ ); ++ { ++ let reader = ArrowArrayStreamReader::from_raw(&mut stream).unwrap(); ++ for batch in reader { ++ let batch = batch.unwrap(); ++ assert_eq!(batch.num_columns(), 1); ++ ids.extend( ++ batch ++ .column(0) ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values() ++ .iter() ++ .copied(), ++ ); ++ } ++ } ++ lance_scanner_close(scanner); ++ lance_dataset_close(ds); ++ } ++ (ids, captured) ++} ++ ++#[test] ++fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { ++ for kind in [ ++ lance_index::IndexType::BTree, ++ lance_index::IndexType::Bitmap, ++ ] { ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(kind, false); ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![2, 3]); ++ assert_eq!(stats.calls, 1); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| name == "scalar_segments_searched" && *value == 1) ++ ); ++ let (ids, _) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); ++ assert_eq!( ++ ids, ++ vec![3], ++ "offset and limit must apply after residual filtering" ++ ); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[1], &[1], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![5, 6, 7]); ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0, 2], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!( ++ ids, ++ vec![2, 3, 9, 10, 11], ++ "partial coverage must not omit unindexed rows" ++ ); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, _)| name == "scalar_segment_fallback_partial_coverage") ++ ); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key = 99 OR id = 0", None, 0); ++ assert_eq!( ++ ids, ++ vec![0], ++ "OR must not use just one branch as candidates" ++ ); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, _)| name == "scalar_segment_fallback_no_driver") ++ ); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key = 99", None, 0); ++ assert!(ids.is_empty()); ++ } ++} ++ ++#[test] ++fn test_scalar_segment_stable_row_ids_and_deletes() { ++ use lance::index::DatasetIndexExt; ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, true); ++ lance_c::runtime::block_on(async { ++ let mut ds = Dataset::open(&uri).await.unwrap(); ++ ds.delete("id = 2").await.unwrap(); ++ assert_eq!(ds.load_indices().await.unwrap().len(), 2); ++ }); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![3]); ++} ++ ++#[test] ++fn test_scalar_segment_requires_explicit_domain_and_checks_uuid() { ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false); ++ let uri = c_str(&uri); ++ let filter = c_str("key >= 0"); ++ unsafe { ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(ptr::null_mut(), ptr::null()), ++ -1 ++ ); ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuids[0].as_ptr()), ++ 0 ++ ); ++ let mut batch = ptr::null_mut(); ++ assert_eq!(lance_scanner_next(scanner, &mut batch), -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ lance_scanner_close(scanner); ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, [0u64].as_ptr(), 1), ++ 0 ++ ); ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, [0u8; 16].as_ptr()), ++ 0 ++ ); ++ assert_eq!(lance_scanner_next(scanner, &mut batch), -1); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ lance_scanner_close(scanner); ++ lance_dataset_close(ds); ++ } ++} + +From 17240674d739293a494c07dfe5aec92a97185363 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Tue, 8 Sep 2026 23:17:23 +0800 +Subject: [PATCH 2/5] update + +--- + docs/scalar-segment-scans.md | 3 ++ + include/lance/lance.h | 4 +-- + src/scalar_segment.rs | 7 +++++ + tests/c_api_test.rs | 60 ++++++++++++++++++++++++++++++++++-- + 4 files changed, 70 insertions(+), 4 deletions(-) + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +index dc7c141..f72dbfc 100644 +--- a/docs/scalar-segment-scans.md ++++ b/docs/scalar-segment-scans.md +@@ -36,6 +36,9 @@ An unknown UUID, absent fragment or invalid option combination is an error. A + known segment with incomplete/unknown coverage, no suitable driver, unsupported + index type, nested key, overlays, fragment reuse, non-exact results or unsupported + row-ID domain falls back to a non-indexed scan of the entire explicit domain. ++Legacy (v1) storage also takes this fallback because ordinary scans cannot consume ++external row masks; it reports `scalar_segment_fallback_legacy_storage` without ++searching the index. + I/O and corruption errors are propagated, not converted to empty results or + successful fallback. + +diff --git a/include/lance/lance.h b/include/lance/lance.h +index cbd8330..ab15247 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1870,8 +1870,8 @@ int32_t lance_scanner_set_index_segments( + * + * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter + * on the selected logical index. All predicates are reapplied during candidate +- * reads; other scalar indices are disabled. OR/NOT-only filters, overlays, +- * fragment reuse, unsupported index types / result domains ++ * reads; other scalar indices are disabled. Legacy storage, OR/NOT-only filters, ++ * overlays, fragment reuse, unsupported index types / result domains + * and missing coverage use the same domain without an index. No filter also + * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the + * unfiltered candidate set. Vector/FTS queries are rejected. +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 2ab1297..5e121c9 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -138,6 +138,13 @@ impl PreparedScalarSegment { + self.dataset.schema().field_by_id(field_id).ok_or_else(|| { + invalid("scalar segment key field is absent from the dataset schema") + })?; ++ // Match Lance's plain-scan external-mask restriction. Keep the scoped, ++ // full-filtered reader intact and avoid index work on legacy storage. ++ if self.dataset.manifest().data_storage_format.lance_file_format() ++ == lance_file::version::ConcreteFileVersion::V1 ++ { ++ return Ok(Some("legacy_storage")); ++ } + // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove + // the field path of an evolved or nested schema. + if !self +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index a8a7eab..0cb998b 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12519,6 +12519,15 @@ fn test_add_columns_stream_null_dataset_consumes_stream() { + fn create_scalar_segment_fixture( + kind: lance_index::IndexType, + stable: bool, ++) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { ++ create_scalar_segment_fixture_with_options(kind, stable, None, &[&[0], &[1]]) ++} ++ ++fn create_scalar_segment_fixture_with_options( ++ kind: lance_index::IndexType, ++ stable: bool, ++ storage_version: Option, ++ segment_fragments: &[&[u32]], + ) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { + use lance::dataset::WriteParams; + use lance::index::DatasetIndexExt; +@@ -12548,6 +12557,7 @@ fn create_scalar_segment_fixture( + Some(WriteParams { + max_rows_per_file: 4, + enable_stable_row_ids: stable, ++ data_storage_version: storage_version, + ..Default::default() + }), + ) +@@ -12561,11 +12571,11 @@ fn create_scalar_segment_fixture( + let fragments = ds.get_fragments(); + assert_eq!(fragments.len(), 3); + let mut segments = Vec::new(); +- for fragment in fragments.iter().take(2) { ++ for fragment_ids in segment_fragments { + segments.push( + ds.create_index_builder(&["key"], kind, ¶ms) + .name("key_idx".into()) +- .fragments(vec![fragment.id() as u32]) ++ .fragments(fragment_ids.to_vec()) + .execute_uncommitted() + .await + .unwrap(), +@@ -12714,6 +12724,52 @@ fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { + } + } + ++#[test] ++fn test_scalar_segment_legacy_storage_falls_back() { ++ use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; ++ ++ // Three fragments, with one segment covering 0 and 1. Reading only fragment ++ // 0 must retain the full predicate and must not leak rows from fragment 1. ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture_with_options( ++ lance_index::IndexType::BTree, ++ false, ++ Some(LanceFileVersion::Legacy), ++ &[&[0, 1]], ++ ); ++ assert_eq!(uuids.len(), 1); ++ lance_c::runtime::block_on(async { ++ let ds = Dataset::open(&uri).await.unwrap(); ++ assert_eq!( ++ ds.manifest().data_storage_format.lance_file_format(), ++ ConcreteFileVersion::V1 ++ ); ++ }); ++ ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![2, 3]); ++ assert_eq!(stats.calls, 1); ++ assert_eq!(stats.indices_loaded, 0); ++ assert_eq!(stats.index_comparisons, 0); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_legacy_storage" && *value == 1 ++ })); ++ assert!( ++ !stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| name == "scalar_segments_searched" && *value != 0) ++ ); ++ ++ let (ids, _) = ++ scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); ++ assert_eq!( ++ ids, ++ vec![3], ++ "fallback must retain LIMIT/OFFSET after filtering" ++ ); ++} ++ + #[test] + fn test_scalar_segment_stable_row_ids_and_deletes() { + use lance::index::DatasetIndexExt; + +From f9ce263e4e32540c26afa6ccd18021ca81c090f1 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Tue, 8 Sep 2026 23:21:02 +0800 +Subject: [PATCH 3/5] update + +--- + src/scalar_segment.rs | 6 +++++- + tests/c_api_test.rs | 6 ++---- + 2 files changed, 7 insertions(+), 5 deletions(-) + +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 5e121c9..01df58b 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -140,7 +140,11 @@ impl PreparedScalarSegment { + })?; + // Match Lance's plain-scan external-mask restriction. Keep the scoped, + // full-filtered reader intact and avoid index work on legacy storage. +- if self.dataset.manifest().data_storage_format.lance_file_format() ++ if self ++ .dataset ++ .manifest() ++ .data_storage_format ++ .lance_file_format() + == lance_file::version::ConcreteFileVersion::V1 + { + return Ok(Some("legacy_storage")); +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 0cb998b..efd4284 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12745,8 +12745,7 @@ fn test_scalar_segment_legacy_storage_falls_back() { + ); + }); + +- let (ids, stats) = +- scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", None, 0); + assert_eq!(ids, vec![2, 3]); + assert_eq!(stats.calls, 1); + assert_eq!(stats.indices_loaded, 0); +@@ -12761,8 +12760,7 @@ fn test_scalar_segment_legacy_storage_falls_back() { + .any(|(name, _, value)| name == "scalar_segments_searched" && *value != 0) + ); + +- let (ids, _) = +- scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0 AND id >= 2", Some(1), 1); + assert_eq!( + ids, + vec![3], + +From 221d8800f56790ab8b48b12f3aaaf9da06359380 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Wed, 9 Sep 2026 12:59:08 +0800 +Subject: [PATCH 4/5] add LabelList + +--- + docs/scalar-segment-scans.md | 12 ++- + include/lance/lance.h | 8 +- + include/lance/lance.hpp | 4 +- + src/scalar_segment.rs | 7 +- + tests/c_api_test.rs | 190 ++++++++++++++++++++++++++++++++--- + 5 files changed, 200 insertions(+), 21 deletions(-) + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +index f72dbfc..72f6188 100644 +--- a/docs/scalar-segment-scans.md ++++ b/docs/scalar-segment-scans.md +@@ -1,10 +1,18 @@ + # Scalar index segment scans + +-An ordinary scanner can use one physical BTree/Bitmap segment to generate +-candidates, then read those candidates with the complete scanner filter. This ++An ordinary scanner can use one physical BTree, Bitmap, or LabelList segment to ++generate candidates, then read them with the complete scanner filter. This + does not run a global search of the other segments of the logical index. It does + not subdivide a physical segment or make its own index search incremental. + ++LabelList supports indexed array membership predicates. Every candidate search ++must return `SearchResult::Exact`. ++LabelList query values should match the array element type, for example ++`array_contains(int32_labels, CAST(42 AS INT))`; a cast on the indexed column ++can prevent the planner from finding an index driver and cause fallback. ++`AtMost` and `AtLeast` results still fall back; this mode does not enable FMIndex, ++NGram, BloomFilter, ZoneMap, or Inverted indices. ++ + ## Configuring a task + + Open a fixed dataset version. Select the physical index UUID from that version's +diff --git a/include/lance/lance.h b/include/lance/lance.h +index ab15247..b5c6902 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1868,9 +1868,11 @@ int32_t lance_scanner_set_index_segments( + * scan of those fragment_ids. Callers distributing work must assign disjoint + * fragment domains and separately include any unindexed data they wish to read. + * +- * BTree/Bitmap searches use a necessary AND-conjunct of the full scanner filter +- * on the selected logical index. All predicates are reapplied during candidate +- * reads; other scalar indices are disabled. Legacy storage, OR/NOT-only filters, ++ * BTree/Bitmap/LabelList searches use a necessary AND-conjunct of the ++ * full scanner filter on the selected logical index and require an Exact result. ++ * AtMost/AtLeast results fall back to a full filtered scan of fragment_ids. ++ * All predicates are reapplied during candidate reads; other scalar indices ++ * are disabled. Legacy storage, OR/NOT-only filters, + * overlays, fragment reuse, unsupported index types / result domains + * and missing coverage use the same domain without an index. No filter also + * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index 8e1c6cf..ebb8141 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1311,8 +1311,8 @@ class Scanner { + return *this; + } + +- /// Restrict scalar candidate generation to one segment; fragment_ids is +- /// required and defines the complete read/fallback domain. See lance.h. ++ /// Generate exact candidates from one BTree/Bitmap/LabelList segment. ++ /// fragment_ids is required and defines the complete read/fallback domain. See lance.h. + Scanner& scalar_index_segment(const std::array& segment_uuid) { + if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0) + check_error(); +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 01df58b..1faf500 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -204,7 +204,12 @@ impl PreparedScalarSegment { + .dataset + .open_scalar_index(&search.column, &self.segment_uuid, metrics) + .await?; +- if !matches!(index.index_type(), IndexType::BTree | IndexType::Bitmap) { ++ // These implementations can return exact candidates. Keep the runtime ++ // Exact check below: a type alone is not a guarantee for every query. ++ if !matches!( ++ index.index_type(), ++ IndexType::BTree | IndexType::Bitmap | IndexType::LabelList ++ ) { + return Ok(Some("index_type")); + } + // External masks use _rowid, not necessarily physical row addresses. +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index efd4284..3322d42 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12528,6 +12528,21 @@ fn create_scalar_segment_fixture_with_options( + stable: bool, + storage_version: Option, + segment_fragments: &[&[u32]], ++) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { ++ let key = Arc::new(Int32Array::from( ++ (0..12) ++ .map(|id| if id % 4 == 0 { None } else { Some(id % 3) }) ++ .collect::>(), ++ )); ++ create_scalar_segment_fixture_from_key(kind, stable, storage_version, segment_fragments, key) ++} ++ ++fn create_scalar_segment_fixture_from_key( ++ kind: lance_index::IndexType, ++ stable: bool, ++ storage_version: Option, ++ segment_fragments: &[&[u32]], ++ key: arrow_array::ArrayRef, + ) -> (tempfile::TempDir, String, Vec<[u8; 16]>) { + use lance::dataset::WriteParams; + use lance::index::DatasetIndexExt; +@@ -12537,17 +12552,14 @@ fn create_scalar_segment_fixture_with_options( + let uuids = lance_c::runtime::block_on(async { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), +- Field::new("key", DataType::Int32, true), ++ Field::new("key", key.data_type().clone(), true), + ])); ++ let row_count = key.len(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ +- Arc::new(Int32Array::from_iter_values(0..12)), +- Arc::new(Int32Array::from( +- (0..12) +- .map(|id| if id % 4 == 0 { None } else { Some(id % 3) }) +- .collect::>(), +- )), ++ Arc::new(Int32Array::from_iter_values(0..row_count as i32)), ++ key, + ], + ) + .unwrap(); +@@ -12563,13 +12575,9 @@ fn create_scalar_segment_fixture_with_options( + ) + .await + .unwrap(); +- let params = ScalarIndexParams::for_builtin(if kind == lance_index::IndexType::Bitmap { +- BuiltinIndexType::Bitmap +- } else { +- BuiltinIndexType::BTree +- }); ++ let params = ScalarIndexParams::for_builtin(BuiltinIndexType::try_from(kind).unwrap()); + let fragments = ds.get_fragments(); +- assert_eq!(fragments.len(), 3); ++ assert_eq!(fragments.len(), row_count.div_ceil(4)); + let mut segments = Vec::new(); + for fragment_ids in segment_fragments { + segments.push( +@@ -12724,6 +12732,162 @@ fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { + } + } + ++#[test] ++fn test_scalar_segment_label_list_exact_candidates() { ++ use arrow_array::builder::{Int32Builder, ListBuilder}; ++ use lance::index::DatasetIndexExt; ++ use lance_index::IndexType; ++ ++ for stable in [false, true] { ++ let mut lists = ListBuilder::new(Int32Builder::new()); ++ for row in 0..16 { ++ match row { ++ 0 | 9 | 13 => lists.append(false), ++ 1 | 10 | 14 => lists.append(true), ++ 4 => { ++ lists.values().append_value(7); ++ lists.append(true); ++ } ++ _ => { ++ lists.values().append_value(42); ++ if row == 3 || row == 11 || row == 15 { ++ lists.values().append_value(7); ++ } ++ if row == 6 { ++ lists.values().append_null(); ++ } ++ lists.append(true); ++ } ++ } ++ } ++ // S0 covers fragments 0 and 1, S1 covers 2, and 3 is unindexed. ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture_from_key( ++ IndexType::LabelList, ++ stable, ++ None, ++ &[&[0, 1], &[2]], ++ Arc::new(lists.finish()), ++ ); ++ let predicate = "array_contains(key, CAST(42 AS INT))"; ++ let filter = format!("{predicate} AND id >= 3"); ++ for (fragments, expected) in [(vec![0, 1], vec![3, 5, 6, 7]), (vec![0], vec![3])] { ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &fragments, &filter, None, 0); ++ assert_eq!(ids, expected, "stable={stable}"); ++ assert_eq!(stats.calls, 1); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| { name == "scalar_segments_searched" && *value == 1 }), ++ "stable={stable}, metrics={:?}", ++ stats.metrics ++ ); ++ assert!( ++ !stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| { name == "scalar_segment_fallbacks" && *value != 0 }) ++ ); ++ } ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], &filter, Some(1), 1); ++ assert_eq!(ids, vec![5], "limit/offset must follow the residual filter"); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[1], &[2], &filter, None, 0); ++ assert_eq!(ids, vec![8, 11]); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0, 3], &filter, None, 0); ++ assert_eq!(ids, vec![3, 12, 15]); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_partial_coverage" && *value == 1 ++ })); ++ let (ids, stats) = scalar_segment_ids( ++ &uri, ++ &uuids[0], ++ &[0], ++ &format!("{predicate} OR id = 0"), ++ None, ++ 0, ++ ); ++ assert_eq!(ids, vec![0, 2, 3]); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_no_driver" && *value == 1 ++ })); ++ ++ for (predicate, expected) in [ ++ ( ++ "array_has_all(key, [CAST(42 AS INT), CAST(7 AS INT)])", ++ vec![3], ++ ), ++ ( ++ "array_has_any(key, [CAST(42 AS INT), CAST(99 AS INT)])", ++ vec![2, 3, 5, 6, 7], ++ ), ++ ("array_contains(key, CAST(99 AS INT))", vec![]), ++ ("array_contains(key, CAST(NULL AS INT))", vec![]), ++ ("array_has_any(key, [])", vec![]), ++ ] { ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], predicate, None, 0); ++ assert_eq!(ids, expected, "{predicate}, stable={stable}"); ++ } ++ // An untyped integer literal casts this Int32 list to Int64. Such ++ // a column expression must retain the scan fallback. ++ let (ids, stats) = ++ scalar_segment_ids(&uri, &uuids[0], &[0, 1], "array_contains(key, 42)", None, 0); ++ assert_eq!(ids, vec![2, 3, 5, 6, 7]); ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_no_driver" && *value == 1 ++ })); ++ ++ lance_c::runtime::block_on(async { ++ let mut ds = Dataset::open(&uri).await.unwrap(); ++ ds.delete("id = 3").await.unwrap(); ++ assert_eq!(ds.load_indices().await.unwrap().len(), 2); ++ }); ++ let (ids, _) = scalar_segment_ids(&uri, &uuids[0], &[0, 1], &filter, None, 0); ++ assert_eq!(ids, vec![5, 6, 7]); ++ } ++} ++ ++#[test] ++fn test_scalar_segment_text_indices_still_fall_back() { ++ for kind in [lance_index::IndexType::Fm, lance_index::IndexType::NGram] { ++ for stable in [false, true] { ++ let key = Arc::new(StringArray::from(vec![ ++ Some("needle"), ++ None, ++ Some(""), ++ Some("other"), ++ Some("needle"), ++ Some("other"), ++ Some(""), ++ None, ++ ])); ++ let (_tmp, uri, uuids) = ++ create_scalar_segment_fixture_from_key(kind, stable, None, &[&[0, 1]], key); ++ for (predicate, expected) in [ ++ ("contains(key, 'needle')", vec![0]), ++ ("contains(key, '')", vec![0, 2, 3]), ++ ] { ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], predicate, None, 0); ++ assert_eq!(ids, expected, "{kind:?}, stable={stable}, {predicate}"); ++ assert!( ++ stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallbacks" && *value == 1 ++ }) ++ ); ++ if predicate == "contains(key, 'needle')" { ++ assert!(stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segment_fallback_index_type" && *value == 1 ++ })); ++ } ++ assert!( ++ !stats.metrics.iter().any(|(name, _, value)| { ++ name == "scalar_segments_searched" && *value != 0 ++ }) ++ ); ++ } ++ } ++ } ++} ++ + #[test] + fn test_scalar_segment_legacy_storage_falls_back() { + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + +From 30dda06a1cc6bad37d04ed207115409ccef91715 Mon Sep 17 00:00:00 2001 +From: zhangstar333 +Date: Wed, 9 Sep 2026 13:27:54 +0800 +Subject: [PATCH 5/5] update + +--- + docs/scalar-segment-scans.md | 55 +++++++++-- + include/lance/lance.h | 16 ++- + include/lance/lance.hpp | 5 + + src/scalar_segment.rs | 12 ++- + src/scanner.rs | 13 ++- + tests/c_api_test.rs | 187 +++++++++++++++++++++++++++++++++++ + 6 files changed, 272 insertions(+), 16 deletions(-) + +diff --git a/docs/scalar-segment-scans.md b/docs/scalar-segment-scans.md +index 72f6188..efd0498 100644 +--- a/docs/scalar-segment-scans.md ++++ b/docs/scalar-segment-scans.md +@@ -27,11 +27,19 @@ lance_scanner_set_limit(scanner, 20000); + /* The scanner-owning thread calls lance_scanner_next as usual. */ + ``` + ++The segment setter copies the UUID; passing NULL clears it. Final option ++compatibility and snapshot metadata are checked when preparing the stream, not ++by the setter. Configure all options before the first `next`, Arrow stream ++export, or asynchronous scan. A failed preparation also freezes the options; ++create a new scanner to retry with different settings. ++ + SQL, Substrait and additional SQL filters keep their existing precedence and AND + composition. The caller does not supply a separate driver predicate: Lance-C + uses the typed filter planner and selects a necessary indexed leaf belonging to + the requested logical index. It only descends through AND, never through OR or +-NOT. It then searches the selected UUID and applies the complete filter while ++NOT, and chooses the first matching leaf in the planner's expression tree; ++this is not a selectivity-based choice or a guarantee of SQL text order. ++It then searches the selected UUID and applies the complete filter while + reading candidates with automatic scalar-index planning disabled. + + Each task's fragment IDs define its result domain, including on fallback. A +@@ -40,9 +48,16 @@ scan. Unindexed fragments need their own tasks, or an explicit domain including + them (which causes that task to use fallback). Merely listing indexed segments + does not include appended, unindexed data automatically. + +-An unknown UUID, absent fragment or invalid option combination is an error. A +-known segment with incomplete/unknown coverage, no suitable driver, unsupported +-index type, nested key, overlays, fragment reuse, non-exact results or unsupported ++`use_scalar_index=false` disables segment search regardless of setter order. ++The scanner validates the selected snapshot UUID and fragment domain, then scans ++that domain with the full filter and LIMIT/OFFSET without opening the index or ++generating candidates. It reports `scalar_segment_fallback_disabled`. ++ ++An unknown UUID, absent fragment, invalid option combination or segment metadata ++without one valid schema key field is an error. A known segment with ++incomplete/unknown coverage, no suitable driver, unsupported ++index type, nested key, overlays, unknown physical row counts, fragment reuse, ++non-exact results or unsupported + row-ID domain falls back to a non-indexed scan of the entire explicit domain. + Legacy (v1) storage also takes this fallback because ordinary scans cannot consume + external row masks; it reports `scalar_segment_fallback_legacy_storage` without +@@ -50,8 +65,15 @@ searching the index. + I/O and corruption errors are propagated, not converted to empty results or + successful fallback. + +-The first implementation supports live-row ordinary scans and cannot be combined +-with vector/FTS queries. Physical row-address ++The first implementation supports live-row ordinary scans and rejects vector/FTS ++queries and `include_deleted_rows=true` at stream creation, even when ++`use_scalar_index=false`. An index built after a delete does not contain the ++tombstoned rows, so even exact segment candidates ++cannot satisfy a scan that includes deleted rows. To read those rows, clear the ++segment setting and use an ordinary scan with `with_row_id=true`, ++`include_deleted_rows=true`, and `use_scalar_index=false`. ++Fragments removed from the current snapshot are not scanned by this option. ++Physical row-address + results on stable-row-ID datasets currently fall back; results already expressed + in the correct row-ID domain use the candidate path. Deletes and all remaining + predicates are handled by the ordinary reader. No candidate-count limit is +@@ -84,6 +106,23 @@ Successful exhaustion merges segment-search metrics into the existing statistics + callback exactly once. New metrics include `scalar_segments_requested`, + `scalar_segments_searched`, `scalar_segment_candidate_rows`, + `scalar_segment_prepare_time`, `scalar_segment_search_time`, and +-`scalar_segment_fallback_*` reasons. `prepare_time` includes search time. Early +-release, cancellation and errors retain the existing callback contract: final ++`scalar_segment_fallbacks` plus `scalar_segment_fallback_*` reasons. ++`scalar_segment_prepare_time` includes search time. Metrics describe each ++successfully exhausted stream, including separately exported streams: ++ ++- `scalar_segments_requested` is 1 even on fallback. ++- `scalar_segments_searched` is 1 after a completed index search, including one ++ whose inexact result causes fallback. A fallback can therefore include index work. ++- `scalar_segment_candidate_rows` counts TRUE rows in the exact segment result before fragment ++ restriction, residual filtering, deletion handling and LIMIT/OFFSET. It is not ++ the output or physical-read row count; a result without a known cardinality is ++ reported as 0. The reader's mask may also include NULL candidates that the full ++ filter subsequently discards. ++- A fallback records one reason, the first eligibility check that fails. Disabled ++ scalar indices and legacy storage bypass index opening and search after snapshot ++ validation. Other fallback reasons may be found after opening or searching an index. ++- Search and candidate metrics may be absent when their stage did not execute; ++ consumers should treat absent counts as 0. ++ ++Early release, cancellation and errors retain the existing callback contract: final + statistics are not guaranteed. Metrics do not establish global task concurrency. +diff --git a/include/lance/lance.h b/include/lance/lance.h +index b5c6902..ed5bb6c 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -1015,7 +1015,9 @@ int32_t lance_scanner_set_scan_in_order(LanceScanner* scanner, bool scan_in_orde + * Configure whether scalar indices may be used to optimize filters. + * + * Scalar indices are enabled by default. Disable this to force filter +- * evaluation without scalar indices. This setting is independent of ++ * evaluation without scalar indices, including an explicitly selected scalar ++ * segment (which falls back to a scan of its explicit fragment_ids). ++ * This setting is independent of + * `lance_scanner_set_use_index`, which controls vector ANN index usage. + * Must be set before scanning starts. + */ +@@ -1051,7 +1053,11 @@ int32_t lance_scanner_with_row_address(LanceScanner* scanner, bool enable); + + /** + * Configure whether deleted rows still present in storage are returned. +- * Deleted rows have a NULL `_rowid`; callers should also enable row IDs. ++ * Requires with_row_id=true; deleted rows have a NULL `_rowid`. ++ * For filtered scans, also set use_scalar_index=false: indices built after a ++ * deletion may omit tombstoned rows. Incompatible with scalar_index_segment, ++ * even when scalar indices are disabled. ++ * Fragments removed from the current snapshot are not scanned. + * Must be set before scanning starts. + */ + int32_t lance_scanner_set_include_deleted_rows( +@@ -1867,16 +1873,20 @@ int32_t lance_scanner_set_index_segments( + * excluded by fragment_ids; incomplete coverage falls back to a full filtered + * scan of those fragment_ids. Callers distributing work must assign disjoint + * fragment domains and separately include any unindexed data they wish to read. ++ * The segment metadata must identify one key field present in the schema. + * + * BTree/Bitmap/LabelList searches use a necessary AND-conjunct of the + * full scanner filter on the selected logical index and require an Exact result. ++ * use_scalar_index=false skips segment search and uses the scoped fallback; ++ * snapshot UUID and fragment validation still applies. + * AtMost/AtLeast results fall back to a full filtered scan of fragment_ids. + * All predicates are reapplied during candidate reads; other scalar indices + * are disabled. Legacy storage, OR/NOT-only filters, + * overlays, fragment reuse, unsupported index types / result domains + * and missing coverage use the same domain without an index. No filter also + * falls back. LIMIT/OFFSET apply after the complete scanner filter, never to the +- * unfiltered candidate set. Vector/FTS queries are rejected. ++ * unfiltered candidate set. Vector/FTS queries and include_deleted_rows=true ++ * are rejected even when use_scalar_index=false; segment mode is live-row-only. + * + * UUID bytes are copied. Metadata and final option compatibility are validated + * when creating the stream. Index corruption or I/O failures remain errors. +diff --git a/include/lance/lance.hpp b/include/lance/lance.hpp +index ebb8141..d96cf9f 100644 +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -1270,6 +1270,7 @@ class Scanner { + } + + /// Configure whether scalar indices may be used to optimize filters. ++ /// False also disables explicit scalar segment search, retaining its fragment domain. + Scanner& use_scalar_index(bool enable = true) { + if (lance_scanner_set_use_scalar_index(handle_.get(), enable) != 0) + check_error(); +@@ -1305,6 +1306,8 @@ class Scanner { + } + + /// Configure whether deleted rows still present in storage are returned. ++ /// Requires with_row_id(true); use_scalar_index(false) is needed for filtered scans. ++ /// Incompatible with scalar_index_segment. See lance.h. + Scanner& include_deleted_rows(bool include_deleted_rows = true) { + if (lance_scanner_set_include_deleted_rows(handle_.get(), include_deleted_rows) != 0) + check_error(); +@@ -1313,6 +1316,8 @@ class Scanner { + + /// Generate exact candidates from one BTree/Bitmap/LabelList segment. + /// fragment_ids is required and defines the complete read/fallback domain. See lance.h. ++ /// Requires live rows only: include_deleted_rows(true) is rejected at stream creation. ++ /// use_scalar_index(false) selects the scoped fallback without searching the segment. + Scanner& scalar_index_segment(const std::array& segment_uuid) { + if (lance_scanner_set_scalar_index_segment(handle_.get(), segment_uuid.data()) != 0) + check_error(); +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 1faf500..747dac6 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -27,6 +27,7 @@ pub(crate) struct PreparedScalarSegment { + pub dataset: Arc, + pub segment_uuid: Uuid, + pub fragment_ids: Vec, ++ pub use_scalar_index: bool, + pub callback: Option, + } + +@@ -138,6 +139,11 @@ impl PreparedScalarSegment { + self.dataset.schema().field_by_id(field_id).ok_or_else(|| { + invalid("scalar segment key field is absent from the dataset schema") + })?; ++ // Explicitly disabling scalar indices also disables this accelerator. ++ // Keep snapshot validation above, but do not plan, open or search an index. ++ if !self.use_scalar_index { ++ return Ok(Some("disabled")); ++ } + // Match Lance's plain-scan external-mask restriction. Keep the scoped, + // full-filtered reader intact and avoid index work on legacy storage. + if self +@@ -149,8 +155,8 @@ impl PreparedScalarSegment { + { + return Ok(Some("legacy_storage")); + } +- // Keep V1 to flat scalar fields. A dotted name is not sufficient to prove +- // the field path of an evolved or nested schema. ++ // Keep this implementation to flat scalar fields. A dotted name cannot ++ // prove the field path of an evolved or nested schema. + if !self + .dataset + .schema() +@@ -234,6 +240,8 @@ impl PreparedScalarSegment { + ); + // Do not truncate candidates at LIMIT. The reader evaluates the complete + // filter before applying its existing limit/offset operators. ++ // The raw selected bitmap can overlap NULL rows; the full filter removes ++ // those as well. The metric above counts semantic TRUE rows, not mask size. + reader.with_row_addr_prefilter(RowAddrMask::from_allowed(rows.selected_rows().clone())); + Ok(None) + } +diff --git a/src/scanner.rs b/src/scanner.rs +index ac3cff6..bda554d 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -479,9 +479,10 @@ impl LanceScanner { + || self.fts_context.is_some() + || self.index_segments.is_some() + || self.fts_index_segments.is_some() ++ || self.include_deleted_rows + { + return Err(lance_core::Error::invalid_input_source( +- "scalar_index_segment requires an ordinary scan of live rows".into(), ++ "scalar_index_segment requires an ordinary scan of live rows; vector/FTS queries and include_deleted_rows=true are unsupported".into(), + )); + } + let fragment_ids = self.fragment_ids.as_ref().filter(|ids| !ids.is_empty()) +@@ -492,6 +493,7 @@ impl LanceScanner { + dataset: Arc::clone(&self.dataset), + segment_uuid, + fragment_ids: fragment_ids.clone(), ++ use_scalar_index: self.use_scalar_index.unwrap_or(true), + callback: self.scan_statistics_callback.clone(), + }) + } else { +@@ -896,6 +898,8 @@ macro_rules! scanner_ffi_try { + + /// Select one physical scalar index segment. NULL clears the selection. + /// Requires explicit fragment_ids and an ordinary live-row scan. See the C header. ++/// include_deleted_rows=true is rejected when preparing the scan, even if ++/// use_scalar_index=false selects the scoped non-indexed fallback. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_scalar_index_segment( + scanner: *mut LanceScanner, +@@ -1239,7 +1243,8 @@ unsafe fn scanner_set_scan_in_order_inner( + /// Configure whether scalar indices may be used to optimize filters. + /// + /// Scalar indices are enabled by default in Lance. Must be set before the scan +-/// starts. ++/// starts. False also disables explicit scalar segment search while preserving ++/// the configured fragment domain and snapshot validation. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_use_scalar_index( + scanner: *mut LanceScanner, +@@ -1381,7 +1386,9 @@ unsafe fn scanner_with_row_address_inner(scanner: *mut LanceScanner, enable: boo + + /// Configure whether deleted rows still present in storage are returned. + /// +-/// Deleted rows have a NULL `_rowid`, so callers should also enable row IDs. ++/// Requires with_row_id=true; deleted rows have a NULL `_rowid`. ++/// Filtered scans also need use_scalar_index=false because indices may omit ++/// tombstoned rows. Incompatible with scalar_index_segment. + /// Must be set before the scan starts. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn lance_scanner_set_include_deleted_rows( +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index 3322d42..513f8d0 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -12945,6 +12945,193 @@ fn test_scalar_segment_stable_row_ids_and_deletes() { + assert_eq!(ids, vec![3]); + } + ++#[test] ++fn test_scalar_segment_honors_use_scalar_index_false() { ++ let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false); ++ let (ids, stats) = scalar_segment_ids(&uri, &uuids[0], &[0], "key >= 0", None, 0); ++ assert_eq!(ids, vec![1, 2, 3]); ++ assert!( ++ stats ++ .metrics ++ .iter() ++ .any(|(name, _, value)| name == "scalar_segments_searched" && *value == 1) ++ ); ++ ++ let uri = c_str(&uri); ++ unsafe { ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ assert!(!ds.is_null()); ++ for disable_first in [false, true] { ++ for (fragments, filter, limit, offset, expected) in [ ++ (vec![0u64], "key >= 0", None, 0, vec![1, 2, 3]), ++ (vec![0, 2], "key >= 0 AND id >= 2", Some(2), 1, vec![3, 9]), ++ ] { ++ let filter = c_str(filter); ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, fragments.as_ptr(), fragments.len()), ++ 0 ++ ); ++ if disable_first { ++ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0); ++ } ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuids[0].as_ptr()), ++ 0 ++ ); ++ if !disable_first { ++ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0); ++ } ++ if let Some(limit) = limit { ++ assert_eq!(lance_scanner_set_limit(scanner, limit), 0); ++ } ++ assert_eq!(lance_scanner_set_offset(scanner, offset), 0); ++ let mut captured = CapturedScanStatistics::default(); ++ assert_eq!( ++ lance_scanner_set_statistics_callback( ++ scanner, ++ Some(capture_scan_statistics), ++ (&mut captured as *mut CapturedScanStatistics).cast(), ++ ), ++ 0 ++ ); ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ assert_eq!(lance_scanner_to_arrow_stream(scanner, &mut stream), 0); ++ let mut ids = Vec::new(); ++ for batch in ArrowArrayStreamReader::from_raw(&mut stream).unwrap() { ++ let batch = batch.unwrap(); ++ ids.extend_from_slice( ++ batch ++ .column_by_name("id") ++ .unwrap() ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values(), ++ ); ++ } ++ assert_eq!(ids, expected); ++ assert_eq!(captured.calls, 1); ++ for metric in ["scalar_segments_searched", "scalar_segment_candidate_rows"] { ++ assert_eq!( ++ captured ++ .metrics ++ .iter() ++ .filter(|(name, _, _)| name == metric) ++ .map(|(_, _, value)| *value) ++ .sum::(), ++ 0, ++ "{metric}" ++ ); ++ } ++ assert_eq!(captured.indices_loaded, 0); ++ assert_eq!(captured.index_comparisons, 0); ++ assert!(captured.metrics.iter().any(|(name, _, value)| name ++ == "scalar_segment_fallback_disabled" ++ && *value == 1)); ++ lance_scanner_close(scanner); ++ } ++ } ++ lance_dataset_close(ds); ++ } ++} ++ ++#[test] ++fn test_scalar_segment_rejects_include_deleted_rows_after_index_rebuild() { ++ use lance::index::DatasetIndexExt; ++ use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; ++ ++ let (_tmp, uri, _) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false); ++ let uuid = lance_c::runtime::block_on(async { ++ let mut ds = Dataset::open(&uri).await.unwrap(); ++ ds.delete("id = 2").await.unwrap(); ++ ds.drop_index("key_idx").await.unwrap(); ++ // A segment built after the delete cannot return the tombstoned row, ++ // even though its search result is Exact for the indexed live rows. ++ let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); ++ let segment = ds ++ .create_index_builder(&["key"], lance_index::IndexType::BTree, ¶ms) ++ .name("key_idx".into()) ++ .fragments(vec![0]) ++ .execute_uncommitted() ++ .await ++ .unwrap(); ++ let uuid = *segment.uuid.as_bytes(); ++ ds.commit_existing_index_segments("key_idx", "key", vec![segment]) ++ .await ++ .unwrap(); ++ uuid ++ }); ++ ++ let (ids, _) = scalar_segment_ids(&uri, &uuid, &[0], "key >= 0 AND id >= 2", None, 0); ++ assert_eq!(ids, vec![3], "live-row segment scans remain supported"); ++ ++ let uri = c_str(&uri); ++ let filter = c_str("key >= 0 AND id >= 2"); ++ unsafe { ++ let ds = lance_dataset_open(uri.as_ptr(), ptr::null(), 0); ++ assert!(!ds.is_null()); ++ // Check both setter orders: compatibility is validated at stream creation. ++ for segment_first in [None, Some(false), Some(true)] { ++ let scanner = lance_scanner_new(ds, ptr::null(), filter.as_ptr()); ++ assert!(!scanner.is_null()); ++ assert_eq!( ++ lance_scanner_set_fragment_ids(scanner, [0u64].as_ptr(), 1), ++ 0 ++ ); ++ assert_eq!(lance_scanner_with_row_id(scanner, true), 0); ++ if segment_first.is_none() { ++ assert_eq!(lance_scanner_set_use_scalar_index(scanner, false), 0); ++ } ++ if segment_first == Some(true) { ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), ++ 0 ++ ); ++ } ++ assert_eq!(lance_scanner_set_include_deleted_rows(scanner, true), 0); ++ if segment_first == Some(false) { ++ assert_eq!( ++ lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), ++ 0 ++ ); ++ } ++ let mut stream = FFI_ArrowArrayStream::empty(); ++ let rc = lance_scanner_to_arrow_stream(scanner, &mut stream); ++ if segment_first.is_some() { ++ assert_eq!(rc, -1, "segment scans must not silently omit deleted rows"); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ assert!(take_last_error_message().contains("include_deleted_rows=true")); ++ } else { ++ assert_eq!(rc, 0); ++ let reader = ArrowArrayStreamReader::from_raw(&mut stream).unwrap(); ++ let mut ids = Vec::new(); ++ for batch in reader { ++ let batch = batch.unwrap(); ++ ids.extend_from_slice( ++ batch ++ .column_by_name("id") ++ .unwrap() ++ .as_any() ++ .downcast_ref::() ++ .unwrap() ++ .values(), ++ ); ++ } ++ ids.sort_unstable(); ++ assert_eq!( ++ ids, ++ vec![2, 3], ++ "ordinary scans can still read tombstoned rows" ++ ); ++ } ++ lance_scanner_close(scanner); ++ } ++ lance_dataset_close(ds); ++ } ++} ++ + #[test] + fn test_scalar_segment_requires_explicit_domain_and_checks_uuid() { + let (_tmp, uri, uuids) = create_scalar_segment_fixture(lance_index::IndexType::BTree, false);