From 367525343c26860989b880e2445816dedbb50ebe Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 10 Sep 2026 16:02:48 +0000 Subject: [PATCH 1/2] Stop DuckDB SLT runs stalling on Vortex scans A Vortex runtime driven from a thread inside `tokio::runtime::Runtime::block_on` loses the wakeups that complete its I/O and parks forever. The DuckDB SLT runner drove each file from inside `rt.block_on`, so a long sequence of scans reliably wedged: tracing showed a read issuing `get_opts` and never being resumed, with every thread parked and the blocking worker already idle. The trigger is narrow and was worth isolating before fixing. It needs both an awaited Tokio task and a driving thread inside `Runtime::block_on`; neither alone reproduces. An `enter()` guard is fine, wakeups from ordinary foreign threads are fine, and a plain `tokio::spawn` stalls just as a `spawn_blocking` does, so this is not about the blocking pool or about worker counts -- a multi-threaded runtime stalls identically. See #9817. DuckDB needs no Tokio here: `AsyncDB::run` is synchronous, `vortex-duckdb` drives its own `CurrentThreadRuntime`, and only DataFusion `.slt` files use the `sleep` and `system` directives. So the DuckDB path now drives its runner with `futures::executor::block_on` and the DataFusion path keeps its Tokio runtime. Also records the real hazard in `resolve_filesystem`, whose comment previously gave only the blocking-pool cost as the reason to keep local files off `Compat`. Fixes #9817 Signed-off-by: "Joe Isaacs" Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01J9w8fY73UoZKRboETTuBit --- Cargo.lock | 1 + vortex-duckdb/src/file_reader.rs | 6 ++++-- vortex-sqllogictest/Cargo.toml | 1 + vortex-sqllogictest/bin/sqllogictests-runner.rs | 13 ++++++++----- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7d3103c9d5..1f81cc58235 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11555,6 +11555,7 @@ dependencies = [ "datafusion 55.0.0", "datafusion-functions-nested 55.0.0", "datafusion-sqllogictest", + "futures", "indicatif", "regex", "rstest", diff --git a/vortex-duckdb/src/file_reader.rs b/vortex-duckdb/src/file_reader.rs index 5048aa8a025..f33a6f869a1 100644 --- a/vortex-duckdb/src/file_reader.rs +++ b/vortex-duckdb/src/file_reader.rs @@ -76,8 +76,10 @@ use crate::table_function::convert_result; // separate thread. fn resolve_filesystem(url: &Url) -> VortexResult<(FileSystemRef, String)> { - // Compat makes us use tokio which is very bad for local reads on - // high-core machines because reads go into blocking pool + // Keep local files off `Compat`, which routes reads through Tokio. Beyond the cost of + // pushing every local read into a blocking pool on high-core machines, a Vortex runtime + // that awaits a Tokio task while its driving thread sits inside `Runtime::block_on` loses + // the completion wakeup and stalls forever (#9817). if url.scheme() == "file" { return Ok(( Arc::new(ObjectStoreFileSystem::local(RUNTIME.handle())), diff --git a/vortex-sqllogictest/Cargo.toml b/vortex-sqllogictest/Cargo.toml index 3033257dfc7..d05d449c14e 100644 --- a/vortex-sqllogictest/Cargo.toml +++ b/vortex-sqllogictest/Cargo.toml @@ -29,6 +29,7 @@ vortex-duckdb = { workspace = true } [dev-dependencies] datafusion = { workspace = true, features = ["parquet"] } datafusion-functions-nested.workspace = true +futures = { workspace = true } indicatif = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } vortex-datafusion = { workspace = true } diff --git a/vortex-sqllogictest/bin/sqllogictests-runner.rs b/vortex-sqllogictest/bin/sqllogictests-runner.rs index 6327473203b..3c56d949633 100644 --- a/vortex-sqllogictest/bin/sqllogictests-runner.rs +++ b/vortex-sqllogictest/bin/sqllogictests-runner.rs @@ -48,11 +48,11 @@ enum Mode { Complete, } -/// Builds a single-threaded Tokio runtime for one test file. +/// Builds a single-threaded Tokio runtime for one DataFusion test file. /// /// `libtest-mimic` runs each trial on its own thread, so a current-thread -/// runtime keeps blocking DuckDB calls and async DataFusion work isolated per -/// file instead of contending for shared multi-threaded runtime workers. +/// runtime keeps async DataFusion work isolated per file instead of contending +/// for shared multi-threaded runtime workers. fn build_runtime() -> anyhow::Result { Ok(tokio::runtime::Builder::new_current_thread() .enable_all() @@ -111,8 +111,11 @@ fn drive_duckdb(path: &Path, work_dir: &Path, mode: Mode) -> anyhow::Result<()> let _guard = WorkDirGuard::new(work_dir.to_path_buf()); let work_dir = work_dir.to_string_lossy().into_owned(); - let rt = build_runtime()?; - rt.block_on(async { + // Deliberately not a Tokio runtime. DuckDB scans drive Vortex's own runtime, and a Vortex + // runtime driven from a thread inside `tokio::runtime::Runtime::block_on` loses the wakeups + // that complete its I/O and stalls forever (#9817). `AsyncDB::run` for DuckDB is synchronous + // and no DuckDB `.slt` uses the `sleep` or `system` directives, so nothing here needs Tokio. + futures::executor::block_on(async { let mut runner = Runner::new(|| async { DuckDB::try_new().map(|db| PathNormalizing::new(db, work_dir.clone())) }); From 9a22cae8698b7411c8256949015f950fac6fa22a Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 10 Sep 2026 17:14:12 +0100 Subject: [PATCH 2/2] wip Signed-off-by: Joe Isaacs --- vortex-duckdb/src/file_reader.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vortex-duckdb/src/file_reader.rs b/vortex-duckdb/src/file_reader.rs index f33a6f869a1..5048aa8a025 100644 --- a/vortex-duckdb/src/file_reader.rs +++ b/vortex-duckdb/src/file_reader.rs @@ -76,10 +76,8 @@ use crate::table_function::convert_result; // separate thread. fn resolve_filesystem(url: &Url) -> VortexResult<(FileSystemRef, String)> { - // Keep local files off `Compat`, which routes reads through Tokio. Beyond the cost of - // pushing every local read into a blocking pool on high-core machines, a Vortex runtime - // that awaits a Tokio task while its driving thread sits inside `Runtime::block_on` loses - // the completion wakeup and stalls forever (#9817). + // Compat makes us use tokio which is very bad for local reads on + // high-core machines because reads go into blocking pool if url.scheme() == "file" { return Ok(( Arc::new(ObjectStoreFileSystem::local(RUNTIME.handle())),