From 37ccbf6d32987a0f3a86e3107aae3d7b30210914 Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:21:48 -0700 Subject: [PATCH] feat(indexes): workspace-wide list with filters and parallel fetch Indexes list no longer requires connection, schema, and table. Defaults to all tables in the workspace; optional flags narrow the catalog scan. Resolve information_schema connection labels to connection IDs via GET /connections. Skip missing tables when the indexes endpoint returns 404 so stale catalog rows do not abort the run. Fetch per-table indexes in parallel with rayon to reduce wall-clock latency. Add ApiClient::Clone and get_none_if_not_found for the scan. --- Cargo.lock | 66 ++++++++++++++-- Cargo.toml | 1 + src/api.rs | 32 ++++++++ src/command.rs | 14 ++-- src/indexes.rs | 205 +++++++++++++++++++++++++++++++++++++++++++++---- src/main.rs | 8 +- 6 files changed, 294 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbebb85..19ac42e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,7 +72,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -83,7 +83,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -248,7 +248,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -317,6 +317,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crossterm" version = "0.28.1" @@ -420,7 +445,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -455,6 +480,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "encode_unicode" version = "1.0.0" @@ -483,7 +514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -717,6 +748,7 @@ dependencies = [ "nix", "open", "rand 0.8.5", + "rayon", "reqwest", "semver", "serde", @@ -1551,6 +1583,26 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1697,7 +1749,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2080,7 +2132,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 14c93ad..e105e69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ anstyle = "1.0.13" clap = { version = "4", features = ["derive"] } directories = "6" reqwest = { version = "0.12", features = ["blocking", "json"] } +rayon = "1.10" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" diff --git a/src/api.rs b/src/api.rs index b200ad4..2db14eb 100644 --- a/src/api.rs +++ b/src/api.rs @@ -4,6 +4,7 @@ use crate::util; use crossterm::style::Stylize; use serde::de::DeserializeOwned; +#[derive(Clone)] pub struct ApiClient { client: reqwest::blocking::Client, api_key: String, @@ -156,6 +157,37 @@ impl ApiClient { } } + /// GET request; returns `None` on HTTP 404. Other status codes use the same handling as + /// [`Self::get`]. Used when probing many paths where a missing resource is normal. + pub fn get_none_if_not_found(&self, path: &str) -> Option { + let url = format!("{}{path}", self.api_url); + self.log_request("GET", &url, None); + + let resp = match self.build_request(reqwest::Method::GET, &url).send() { + Ok(r) => r, + Err(e) => { + eprintln!("error connecting to API: {e}"); + std::process::exit(1); + } + }; + + let (status, body) = util::debug_response(resp); + if status == reqwest::StatusCode::NOT_FOUND { + return None; + } + if !status.is_success() { + self.fail_response(status, body); + } + + match serde_json::from_str(&body) { + Ok(v) => Some(v), + Err(e) => { + eprintln!("error parsing response: {e}"); + std::process::exit(1); + } + } + } + /// POST request with JSON body, returns parsed response. pub fn post(&self, path: &str, body: &serde_json::Value) -> T { let url = format!("{}{path}", self.api_url); diff --git a/src/command.rs b/src/command.rs index 755ad22..3d1cd23 100644 --- a/src/command.rs +++ b/src/command.rs @@ -248,19 +248,19 @@ pub enum AuthCommands { #[derive(Subcommand)] pub enum IndexesCommands { - /// List indexes on a table + /// List indexes (defaults to the whole workspace; narrow with filters) List { - /// Connection ID + /// Filter by connection ID #[arg(long, short = 'c')] - connection_id: String, + connection_id: Option, - /// Schema name + /// Filter by schema name #[arg(long)] - schema: String, + schema: Option, - /// Table name + /// Filter by table name #[arg(long)] - table: String, + table: Option, /// Output format #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])] diff --git a/src/indexes.rs b/src/indexes.rs index c67eff6..42c7eaa 100644 --- a/src/indexes.rs +++ b/src/indexes.rs @@ -1,5 +1,7 @@ use crate::api::ApiClient; +use rayon::prelude::*; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; #[derive(Deserialize, Serialize)] struct Index { @@ -12,39 +14,208 @@ struct Index { updated_at: String, } +#[derive(Serialize)] +struct IndexRow { + #[serde(flatten)] + inner: Index, + #[serde(skip_serializing_if = "Option::is_none")] + table: Option, +} + #[derive(Deserialize)] struct ListResponse { indexes: Vec, } +#[derive(Deserialize)] +struct InfoTable { + connection: String, + schema: String, + table: String, +} + +#[derive(Deserialize)] +struct InfoListResponse { + tables: Vec, + has_more: bool, + next_cursor: Option, +} + +#[derive(Deserialize)] +struct ConnectionRef { + id: String, + name: String, +} + +#[derive(Deserialize)] +struct ConnectionsBody { + connections: Vec, +} + +fn connection_lookup(api: &ApiClient) -> HashMap { + let body: ConnectionsBody = api.get("/connections"); + let mut m = HashMap::new(); + for c in body.connections { + m.insert(c.id.clone(), c.id.clone()); + m.insert(c.name.clone(), c.id); + } + m +} + +fn collect_tables( + api: &ApiClient, + connection_id: Option<&str>, + schema: Option<&str>, + table: Option<&str>, +) -> Vec { + let mut out = Vec::new(); + let mut cursor: Option = None; + loop { + let mut params: Vec<(&str, Option)> = Vec::new(); + if let Some(id) = connection_id { + params.push(("connection_id", Some(id.to_string()))); + } + if let Some(s) = schema { + params.push(("schema", Some(s.to_string()))); + } + if let Some(t) = table { + params.push(("table", Some(t.to_string()))); + } + if let Some(ref c) = cursor { + params.push(("cursor", Some(c.clone()))); + } + let body: InfoListResponse = api.get_with_params("/information_schema", ¶ms); + out.extend(body.tables); + if !body.has_more { + break; + } + cursor = body.next_cursor; + } + out.sort_by(|a, b| { + a.connection + .cmp(&b.connection) + .then_with(|| a.schema.cmp(&b.schema)) + .then_with(|| a.table.cmp(&b.table)) + }); + out +} + +fn list_one_table(api: &ApiClient, connection_id: &str, schema: &str, table: &str) -> Vec { + let path = format!("/connections/{connection_id}/tables/{schema}/{table}/indexes"); + let body: ListResponse = api.get(&path); + body.indexes +} + +fn list_one_table_scan(api: &ApiClient, connection_id: &str, schema: &str, table: &str) -> Vec { + let path = format!("/connections/{connection_id}/tables/{schema}/{table}/indexes"); + match api.get_none_if_not_found::(&path) { + Some(body) => body.indexes, + None => Vec::new(), + } +} + pub fn list( workspace_id: &str, - connection_id: &str, - schema: &str, - table: &str, + connection_id: Option<&str>, + schema: Option<&str>, + table: Option<&str>, format: &str, ) { let api = ApiClient::new(Some(workspace_id)); - let path = format!("/connections/{connection_id}/tables/{schema}/{table}/indexes"); - let body: ListResponse = api.get(&path); + + let (rows, multi_table) = match (connection_id, schema, table) { + (Some(cid), Some(sch), Some(tbl)) => { + let indexes = list_one_table(&api, cid, sch, tbl); + let rows: Vec = indexes + .into_iter() + .map(|i| IndexRow { + inner: i, + table: None, + }) + .collect(); + (rows, false) + } + _ => { + let tables = collect_tables(&api, connection_id, schema, table); + let conn_ids = connection_lookup(&api); + let api = api.clone(); + let per_table: Vec<(String, Vec)> = tables + .par_iter() + .map(|t| { + let cid = conn_ids + .get(&t.connection) + .map(String::as_str) + .unwrap_or(t.connection.as_str()); + let full = format!("{}.{}.{}", t.connection, t.schema, t.table); + let indexes = list_one_table_scan(&api, cid, &t.schema, &t.table); + (full, indexes) + }) + .collect(); + let mut rows: Vec = Vec::new(); + for (full, indexes) in per_table { + for i in indexes { + rows.push(IndexRow { + inner: i, + table: Some(full.clone()), + }); + } + } + (rows, true) + } + }; match format { - "json" => println!("{}", serde_json::to_string_pretty(&body.indexes).unwrap()), - "yaml" => print!("{}", serde_yaml::to_string(&body.indexes).unwrap()), + "json" => println!("{}", serde_json::to_string_pretty(&rows).unwrap()), + "yaml" => print!("{}", serde_yaml::to_string(&rows).unwrap()), "table" => { - if body.indexes.is_empty() { + if rows.is_empty() { use crossterm::style::Stylize; eprintln!("{}", "No indexes found.".dark_grey()); + } else if multi_table { + let table_rows: Vec> = rows + .iter() + .map(|r| { + vec![ + r.table.clone().unwrap_or_default(), + r.inner.index_name.clone(), + r.inner.index_type.clone(), + r.inner.columns.join(", "), + r.inner.metric.clone().unwrap_or_default(), + r.inner.status.clone(), + crate::util::format_date(&r.inner.created_at), + ] + }) + .collect(); + crate::table::print( + &[ + "TABLE", + "NAME", + "TYPE", + "COLUMNS", + "METRIC", + "STATUS", + "CREATED", + ], + &table_rows, + ); } else { - let rows: Vec> = body.indexes.iter().map(|i| vec![ - i.index_name.clone(), - i.index_type.clone(), - i.columns.join(", "), - i.metric.clone().unwrap_or_default(), - i.status.clone(), - crate::util::format_date(&i.created_at), - ]).collect(); - crate::table::print(&["NAME", "TYPE", "COLUMNS", "METRIC", "STATUS", "CREATED"], &rows); + let table_rows: Vec> = rows + .iter() + .map(|r| { + vec![ + r.inner.index_name.clone(), + r.inner.index_type.clone(), + r.inner.columns.join(", "), + r.inner.metric.clone().unwrap_or_default(), + r.inner.status.clone(), + crate::util::format_date(&r.inner.created_at), + ] + }) + .collect(); + crate::table::print( + &["NAME", "TYPE", "COLUMNS", "METRIC", "STATUS", "CREATED"], + &table_rows, + ); } } _ => unreachable!(), diff --git a/src/main.rs b/src/main.rs index 9807a75..4b47770 100644 --- a/src/main.rs +++ b/src/main.rs @@ -251,7 +251,13 @@ fn main() { let workspace_id = resolve_workspace(workspace_id); match command { IndexesCommands::List { connection_id, schema, table, output } => { - indexes::list(&workspace_id, &connection_id, &schema, &table, &output) + indexes::list( + &workspace_id, + connection_id.as_deref(), + schema.as_deref(), + table.as_deref(), + &output, + ) } IndexesCommands::Create { connection_id, schema, table, name, columns, r#type, metric, r#async } => { indexes::create(&workspace_id, &connection_id, &schema, &table, &name, &columns, &r#type, metric.as_deref(), r#async)