From 6711156250088577d386026c2affd145b62aa8b0 Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Tue, 17 Nov 2026 13:57:07 -0600 Subject: [PATCH 1/8] Open console query results in Positron's Data Explorer Console query results get a link that opens them in Positron's Data Explorer. The kernel keeps each result in a dedicated in-memory DuckDB results cache (sharing `CacheTables` with `CachingReader`, with pinning so open results are not evicted; the user's reader is never wrapped) and exposes a frontend-agnostic results API over `call_method`. The VS Code extension registers the Data Explorer RPC handler, adds the console link, and pages, sorts, filters and profiles results through that API. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 + Cargo.lock | 1 + Cargo.toml | 2 +- doc/get_started/tooling/positron-vscode.qmd | 5 + ggsql-jupyter/AGENTS.md | 24 +- ggsql-jupyter/Cargo.toml | 5 +- ggsql-jupyter/examples/results_bench.rs | 287 ++++++++ ggsql-jupyter/src/display.rs | 130 +++- ggsql-jupyter/src/kernel.rs | 89 ++- ggsql-jupyter/src/lib.rs | 1 + ggsql-jupyter/src/main.rs | 26 + ggsql-jupyter/src/results/export.rs | 108 +++ ggsql-jupyter/src/results/filter.rs | 472 +++++++++++++ ggsql-jupyter/src/results/methods.rs | 246 +++++++ ggsql-jupyter/src/results/mod.rs | 729 ++++++++++++++++++++ ggsql-jupyter/src/results/view.rs | 686 ++++++++++++++++++ ggsql-jupyter/tests/test_integration.py | 153 +++- ggsql-vscode/AGENTS.md | 33 +- ggsql-vscode/package.json | 9 +- ggsql-vscode/src/dataExplorer/filters.ts | 156 +++++ ggsql-vscode/src/dataExplorer/format.ts | 57 ++ ggsql-vscode/src/dataExplorer/handler.ts | 503 ++++++++++++++ ggsql-vscode/src/dataExplorer/profiles.ts | 152 ++++ ggsql-vscode/src/dataExplorer/protocol.ts | 267 +++++++ ggsql-vscode/src/extension.ts | 10 +- ggsql-vscode/src/manager.ts | 36 +- ggsql-vscode/src/resultLinks.ts | 150 ++++ ggsql-vscode/src/test/dataExplorer.test.ts | 390 +++++++++++ ggsql-vscode/src/test/manager.test.ts | 13 +- ggsql-vscode/src/test/resultLinks.test.ts | 132 ++++ src/AGENTS.md | 2 +- src/reader/cache.rs | 406 +++++++---- src/reader/connection.rs | 19 +- src/reader/duckdb.rs | 66 ++ src/reader/odbc/mod.rs | 2 +- 35 files changed, 5185 insertions(+), 190 deletions(-) create mode 100644 ggsql-jupyter/examples/results_bench.rs create mode 100644 ggsql-jupyter/src/results/export.rs create mode 100644 ggsql-jupyter/src/results/filter.rs create mode 100644 ggsql-jupyter/src/results/methods.rs create mode 100644 ggsql-jupyter/src/results/mod.rs create mode 100644 ggsql-jupyter/src/results/view.rs create mode 100644 ggsql-vscode/src/dataExplorer/filters.ts create mode 100644 ggsql-vscode/src/dataExplorer/format.ts create mode 100644 ggsql-vscode/src/dataExplorer/handler.ts create mode 100644 ggsql-vscode/src/dataExplorer/profiles.ts create mode 100644 ggsql-vscode/src/dataExplorer/protocol.ts create mode 100644 ggsql-vscode/src/resultLinks.ts create mode 100644 ggsql-vscode/src/test/dataExplorer.test.ts create mode 100644 ggsql-vscode/src/test/resultLinks.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dd6f53c7..9eae29812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,14 @@ determines whether text boxes get drawn and additional settings `padding`, Data Explorer (#536). - The Positron extension now registers a bundled agent skill, so agents automatically discover how to write and run ggsql queries (#536). +- In a Positron console, a query result can be opened in the Data Explorer + from an "Open in Data Explorer" link under its table, with sorting, + filtering, profiles, export and Convert to Code. Long results show their + first 10 rows in the console, configurable with `ggsql.consoleRows`. The + kernel serves this through a frontend-neutral results API, reached with + `call_method` on the UI comm, which keeps opened results in an in-memory + cache of its own and never changes the session's reader or dialect + (posit-dev/positron#16137). ### Changed diff --git a/Cargo.lock b/Cargo.lock index 3cdb9db58..6968a7332 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1791,6 +1791,7 @@ dependencies = [ "ggsql", "hex", "hmac", + "regex", "serde", "serde_json", "sha2", diff --git a/Cargo.toml b/Cargo.toml index 457247270..b9e1bd2df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ tree-sitter = "0.26" arrow = { version = "58", default-features = false } # Readers -duckdb = { version = "~1.10502", features = ["bundled", "vtab-arrow"] } +duckdb = { version = "~1.10502", features = ["bundled", "vtab-arrow", "appender-arrow"] } parquet = { version = "58", default-features = false, features = ["arrow", "snap"] } bytes = "1" rusqlite = { version = "0.38", features = ["bundled", "chrono", "load_extension"] } diff --git a/doc/get_started/tooling/positron-vscode.qmd b/doc/get_started/tooling/positron-vscode.qmd index 34af05621..0aa5c828e 100644 --- a/doc/get_started/tooling/positron-vscode.qmd +++ b/doc/get_started/tooling/positron-vscode.qmd @@ -30,6 +30,11 @@ Executing ggsql statements containing [`VISUALISE`](../../syntax/clause/visualis ![](./screenshots/positron-plots-pane.png) +## Data Explorer + +A query without `VISUALISE` shows its result as a table in the console, followed by an "Open in Data Explorer" link. A result of more than 20 rows shows only its first 10, with a count of the rest. To show more or fewer, change the `ggsql.consoleRows` setting: a result of up to twice that many rows is shown in full. The change applies to consoles started afterwards. Clicking the link opens the result in Positron's Data Explorer, where you can sort, filter, and profile it, and use "Convert to Code" to turn your sorts and filters back into SQL. + +The Data Explorer works on a snapshot of the result, so it doesn't change when the underlying tables do. Run the query again to see new data. The console keeps the last 16 results available this way; the link under an older one reports that it is no longer available. ## Editor and cells diff --git a/ggsql-jupyter/AGENTS.md b/ggsql-jupyter/AGENTS.md index 4cde53c91..0b7aa6bd1 100644 --- a/ggsql-jupyter/AGENTS.md +++ b/ggsql-jupyter/AGENTS.md @@ -9,6 +9,7 @@ End-user installation and usage live in [`README.md`](README.md). End-user noteb ``` ggsql-jupyter/ ├── Cargo.toml Rust binary + library, depends on ggsql with duckdb + svg/pdf + raster-plots +├── examples/results_bench.rs Results API benchmark (DuckDB cache vs. Arrow row indices) ├── pyproject.toml maturin config (bindings = "bin") for the PyPI wheel ├── README.md User-facing install + usage ├── src/ @@ -17,7 +18,13 @@ ggsql-jupyter/ │ ├── kernel.rs Jupyter messaging loop (ZMQ, message dispatch) │ ├── executor.rs Runs queries via ggsql::Reader, returns rendered output │ ├── connection.rs Reader lifecycle, connection-string parsing -│ ├── data_explorer.rs Positron data-explorer comm channel +│ ├── data_explorer.rs Positron data-explorer comm, for tables opened from the Connections pane +│ ├── results/ Frontend-neutral results API over recent console results +│ │ ├── mod.rs ResultStore: pending results, the results cache, views +│ │ ├── view.rs One view: filters, sort, order table, pages, profiles, export +│ │ ├── filter.rs Filter and SortKey, and the SQL for them +│ │ ├── methods.rs The API as named JSON calls (`result_open`, `result_page`, …) +│ │ └── export.rs A selection as CSV / TSV / HTML │ ├── display.rs Output formatting (image bundles, SQL → HTML table) │ ├── message.rs Jupyter message structs (ZMQ frames, HMAC signing) │ └── util.rs @@ -173,9 +180,22 @@ So `RenderHints::canvas` picks a height (golden ratio, close to ggplot2's defaul ## Positron-specific bits -- `data_explorer.rs` implements Positron's data-explorer comm channel (registered query results become explorable tables). +- `data_explorer.rs` implements Positron's data-explorer comm for a table opened from the Connections pane, querying the session's own database. - The companion VS Code extension (`ggsql-vscode/`) ships a copy of this binary and also discovers installed ones via the `ggsql.kernelPath` setting, the Jupyter kernelspec directories, the native install locations, or `PATH`. It runs each one with `--version` to name the runtime it registers, so **keep `--version` working**: a kernel that does not answer it is still offered, but without a version in the picker. See [Finding the kernel](../ggsql-vscode/AGENTS.md#finding-the-kernel). +### The results API + +`results/` keeps recent **console** query results and answers a small, frontend-neutral set of calls on them (`methods.rs` documents each: `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`). Nothing in it knows about Positron: the ggsql extension translates Positron's Data Explorer protocol into these calls (see [`ggsql-vscode/AGENTS.md`](../ggsql-vscode/AGENTS.md)). The kernel's only Positron-specific part is the route: a `call_method` request on the `positron.ui` comm — what an extension's `session.callMethod(name, params)` sends — whose method starts with `result_` goes to `methods::call`, and its answer or error comes back as the JSON-RPC reply (errors with `plot::comm::RpcError`'s encoding). Any other UI request still gets `result: null`. + +- **Which results are kept.** A tabular result of a non-silent execution in a `PositronConsole` session. Its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data, the newest `--max-results` (default 16). +- **Opening copies the result into the results cache.** A `CacheTables` (the caching reader's bookkeeping, `ggsql::reader::cache`) on its own in-memory DuckDB, bounded by `GGSQL_RESULTS_MAX_BYTES` (default 1 GB). The session's reader is never wrapped, so the user's dialect is untouched. The copy adds `__ggsql_row__`, each row's position in the query's output, and drops the Arrow data, so a result is held once. It stays **pinned** while any view of it is open, so eviction never pulls a table from under an open view; once closed, it is evicted when the cache needs the room, and reopening before then reuses it. +- **Nothing is written to disk.** The cache sets `temp_directory = ''`, so DuckDB cannot spill: a query that needs more memory than the limit fails with an error instead. +- **Sorting and filtering build an order table** of `(pos, __ggsql_row__)` per view (`view.rs`); a page joins it back to the result by `pos` range, so scrolling never sorts again. Ties break on `__ggsql_row__`, so paging is stable and clearing the sort restores the query's own order. An unfiltered, unsorted view has none and pages by `__ggsql_row__` directly. +- **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. +- **`result_sql` writes SQL for the session's database**, not the cache: the result's query as a derived table, with the view's filters and sort, in the session's dialect. A filter it cannot express there is left out with a `--` comment. + +`examples/results_bench.rs` measures it. At 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 1.5 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. + ## Build & install ```sh diff --git a/ggsql-jupyter/Cargo.toml b/ggsql-jupyter/Cargo.toml index 47590aef3..1774a883d 100644 --- a/ggsql-jupyter/Cargo.toml +++ b/ggsql-jupyter/Cargo.toml @@ -33,12 +33,15 @@ arrow = { workspace = true } tokio = { workspace = true, features = ["sync"] } # CLI -clap = { workspace = true } +clap = { workspace = true, features = ["env"] } # Serialization serde = { workspace = true } serde_json = { workspace = true } +# The data explorer's regular-expression column search +regex = { workspace = true } + # Error handling anyhow = { workspace = true } thiserror = { workspace = true } diff --git a/ggsql-jupyter/examples/results_bench.rs b/ggsql-jupyter/examples/results_bench.rs new file mode 100644 index 000000000..37492be1b --- /dev/null +++ b/ggsql-jupyter/examples/results_bench.rs @@ -0,0 +1,287 @@ +//! Benchmark for the results API: the DuckDB results cache it uses, against +//! sorting and filtering the Arrow data directly with row indices. +//! +//! ```sh +//! cargo run --release -p ggsql-jupyter --example results_bench -- store 10000000 +//! cargo run --release -p ggsql-jupyter --example results_bench -- arrow 10000000 +//! ``` +//! +//! Each mode runs in its own process so the resident-memory figures don't mix. +//! `store` also opens and closes a result repeatedly, to check that memory +//! levels off rather than growing with every cycle. + +use arrow::array::{ + Array, ArrayRef, BooleanArray, Date32Array, Float64Array, Int64Array, StringArray, UInt32Array, +}; +use arrow::compute::kernels::cmp::{gt_eq, lt_eq}; +use arrow::compute::kernels::comparison::contains; +use arrow::compute::{and, filter, lexsort_to_indices, take, SortColumn, SortOptions}; +use arrow::datatypes::{DataType, Field, Schema}; +use arrow::record_batch::RecordBatch; +use ggsql::reader::connection::reader_from_uri; +use ggsql::DataFrame; +use ggsql_jupyter::results::filter::Capabilities; +use ggsql_jupyter::results::methods::{self, Session}; +use ggsql_jupyter::results::ResultStore; +use serde_json::{json, Value}; +use std::sync::Arc; +use std::time::Instant; + +const CATEGORIES: &[&str] = &[ + "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", + "kilo", "lima", "mike", "november", "oscar", "papa", "quebec", "romeo", "sierra", "tango", +]; + +struct Lcg(u64); +impl Lcg { + fn next(&mut self) -> u64 { + self.0 = self + .0 + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + self.0 >> 33 + } +} + +fn make_data(n: usize) -> RecordBatch { + let mut rng = Lcg(42); + let id: Int64Array = (0..n as i64).collect(); + let x: Float64Array = (0..n) + .map(|_| { + let v = rng.next(); + (v % 50 != 0).then(|| (v % 1_000_000) as f64 / 7.0) + }) + .collect(); + let category: StringArray = (0..n) + .map(|_| Some(CATEGORIES[(rng.next() % CATEGORIES.len() as u64) as usize])) + .collect(); + let name: StringArray = (0..n) + .map(|_| { + let len = 12 + (rng.next() % 20) as usize; + let s: String = (0..len) + .map(|_| (b'a' + (rng.next() % 26) as u8) as char) + .collect(); + Some(s) + }) + .collect(); + let day: Date32Array = (0..n).map(|_| Some((rng.next() % 20_000) as i32)).collect(); + let schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("x", DataType::Float64, true), + Field::new("category", DataType::Utf8, false), + Field::new("name", DataType::Utf8, false), + Field::new("day", DataType::Date32, false), + ]); + RecordBatch::try_new( + Arc::new(schema), + vec![ + Arc::new(id), + Arc::new(x), + Arc::new(category), + Arc::new(name), + Arc::new(day), + ], + ) + .unwrap() +} + +fn rss_mb() -> f64 { + let out = std::process::Command::new("ps") + .args(["-o", "rss=", "-p", &std::process::id().to_string()]) + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout) + .trim() + .parse::() + .unwrap_or(0.0) + / 1024.0 +} + +fn time(label: &str, f: impl FnOnce() -> T) -> T { + let t = Instant::now(); + let r = f(); + println!( + " {label:<40} {:>9.1} ms rss {:>7.0} MB", + t.elapsed().as_secs_f64() * 1000.0, + rss_mb() + ); + r +} + +/// Sorting and filtering the Arrow data directly, for comparison. +fn run_arrow(batch: RecordBatch) { + let n = batch.num_rows(); + let rows: ArrayRef = Arc::new(UInt32Array::from_iter_values(0..n as u32)); + let col = |name: &str| batch.column(batch.schema().index_of(name).unwrap()).clone(); + let asc = Some(SortOptions { + descending: false, + nulls_first: false, + }); + // Stable sorts: the row index is the last key. + let sort = |keys: &[&str]| { + let mut columns: Vec = keys + .iter() + .map(|k| SortColumn { + values: col(k), + options: asc, + }) + .collect(); + columns.push(SortColumn { + values: rows.clone(), + options: asc, + }); + lexsort_to_indices(&columns, None).unwrap() + }; + + let sort1 = time("sort x", || sort(&["x"])); + time("sort category, x", || sort(&["category", "x"])); + time("sort x (sort_to_indices, unstable)", || { + arrow::compute::sort_to_indices(&col("x"), asc, None).unwrap() + }); + time("filter name contains 'abc'", || { + let name = col("name"); + let name = name.as_any().downcast_ref::().unwrap(); + let mask = contains(name, &StringArray::new_scalar("abc")).unwrap(); + filter(&rows, &mask).unwrap() + }); + time("filter 1000 <= x <= 50000", || { + let x = col("x"); + let mask: BooleanArray = and( + >_eq(&x, &Float64Array::new_scalar(1000.0)).unwrap(), + <_eq(&x, &Float64Array::new_scalar(50000.0)).unwrap(), + ) + .unwrap(); + filter(&rows, &mask).unwrap() + }); + time("page 100 rows mid sorted view", || { + let page = sort1.slice(n / 2, 100); + batch + .columns() + .iter() + .map(|c| take(c, &page, None).unwrap()) + .collect::>() + }); +} + +/// The results API itself, through its JSON calls. +fn run_store(batch: RecordBatch, cycles: usize) { + let n = batch.num_rows(); + let session_reader = reader_from_uri("duckdb://memory").unwrap(); + let session = Session { + reader: &*session_reader, + caps: Capabilities { regex: true }, + }; + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert( + DataFrame::from_record_batch(batch), + "bench".into(), + "SELECT 1".into(), + ); + let mut call = |method: &str, params: Value| { + methods::call(&mut store, &session, method, ¶ms) + .unwrap_or_else(|e| panic!("{method}: {e}")) + }; + + let view = time("open (copy into the cache)", || { + call("result_open", json!({ "key": key })) + })["view"] + .as_str() + .unwrap() + .to_string(); + let sort = |keys: Value| json!({ "view": view, "keys": keys }); + let filters = |f: Value| json!({ "view": view, "filters": f }); + let mid = json!({ "view": view, "rows": {"first": n / 2, "last": n / 2 + 99}, "columns": [0, 1, 2, 3, 4] }); + + time("page 100 rows mid unsorted", || { + call("result_page", mid.clone()) + }); + time("sort x", || { + call("result_set_sort", sort(json!([{"column": 1}]))) + }); + time("page 100 rows mid sorted view", || { + call("result_page", mid.clone()) + }); + time("sort category, x", || { + call( + "result_set_sort", + sort(json!([{"column": 2}, {"column": 1}])), + ) + }); + call("result_set_sort", sort(json!([]))); + time("filter name contains 'abc'", || { + call( + "result_set_filters", + filters( + json!([{"type": "text", "column": 3, "match": "contains", "term": "abc", "case_sensitive": true}]), + ), + ) + }); + time("filter 1000 <= x <= 50000", || { + call( + "result_set_filters", + filters(json!([{"type": "between", "column": 1, "low": "1000", "high": "50000"}])), + ) + }); + time("filter range then sort x", || { + call("result_set_sort", sort(json!([{"column": 1}]))) + }); + time("profile x (summary + histogram)", || { + call( + "result_profile", + json!({"view": view, "column": 1, "profiles": {"null_count": true, "summary": true, "histogram": {"bins": 20, "quantiles": []}}}), + ) + }); + call("result_close", json!({ "view": view })); + println!( + " after close rss {:>7.0} MB", + rss_mb() + ); + + // Reopen, sort, page and close repeatedly: memory must level off. + for cycle in 1..=cycles { + let view = call("result_open", json!({ "key": key }))["view"] + .as_str() + .unwrap() + .to_string(); + call( + "result_set_sort", + json!({ "view": view, "keys": [{"column": 2}, {"column": 1}] }), + ); + call( + "result_set_filters", + json!({ "view": view, "filters": [{"type": "between", "column": 1, "low": "1000", "high": "50000"}] }), + ); + call( + "result_page", + json!({ "view": view, "rows": {"first": 0, "last": 99}, "columns": [0, 1] }), + ); + call("result_close", json!({ "view": view })); + println!( + " cycle {cycle:>2} open/sort/filter/page/close rss {:>7.0} MB", + rss_mb() + ); + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let mode = args.get(1).map(String::as_str).unwrap_or("store"); + let n: usize = args + .get(2) + .and_then(|s| s.parse().ok()) + .unwrap_or(1_000_000); + let cycles: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(10); + println!("mode={mode} rows={n} baseline rss {:.0} MB", rss_mb()); + let batch = time("generate data", || make_data(n)); + let bytes: usize = batch + .columns() + .iter() + .map(|c| c.get_array_memory_size()) + .sum(); + println!(" arrow data size {:.0} MB", bytes as f64 / 1e6); + match mode { + "arrow" => run_arrow(batch), + "store" => run_store(batch, cycles), + _ => panic!("mode must be store or arrow"), + } +} diff --git a/ggsql-jupyter/src/display.rs b/ggsql-jupyter/src/display.rs index f814ddb49..37ce7774b 100644 --- a/ggsql-jupyter/src/display.rs +++ b/ggsql-jupyter/src/display.rs @@ -11,6 +11,7 @@ use base64::engine::general_purpose::STANDARD as BASE64; use base64::Engine; use clap::ValueEnum; use ggsql::reader::ResolvedPlot; +use ggsql::util::escape_html; use ggsql::DataFrame; use serde_json::{json, Value}; @@ -71,14 +72,30 @@ impl SessionKind { } } +/// How many rows of a long result a console shows, unless configured. +pub const DEFAULT_CONSOLE_ROWS: usize = 10; + /// Frontend-supplied hints about the output rendering slot. -#[derive(Default, Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy)] pub struct RenderHints { pub kind: SessionKind, /// Width of the output slot in CSS pixels, when the frontend says. pub output_width_px: Option, /// Device pixel ratio of the display, when the frontend says. pub pixel_ratio: Option, + /// How many rows of a long result a console shows (`--console-rows`). + pub console_rows: usize, +} + +impl Default for RenderHints { + fn default() -> Self { + Self { + kind: SessionKind::default(), + output_width_px: None, + pixel_ratio: None, + console_rows: DEFAULT_CONSOLE_ROWS, + } + } } impl RenderHints { @@ -102,6 +119,7 @@ impl RenderHints { kind: SessionKind::resolve(header.session.as_str(), mode), output_width_px, pixel_ratio, + ..Self::default() } } @@ -175,7 +193,7 @@ pub fn format_display_data( if df.width() == 0 { Ok(Formatted::Nothing) } else { - Ok(Formatted::Bundle(format_dataframe(df))) + Ok(Formatted::Bundle(format_dataframe(df, hints))) } } ExecutionResult::ConnectionChanged { display_name, .. } => { @@ -256,8 +274,8 @@ fn format_static( } /// Format DataFrame as HTML table -fn format_dataframe(df: DataFrame) -> Value { - let html = dataframe_to_html(&df); +fn format_dataframe(df: DataFrame, hints: &RenderHints) -> Value { + let html = dataframe_to_html(&df, html_rows_shown(df.height(), hints)); let text = dataframe_to_text(&df); json!({ @@ -270,11 +288,38 @@ fn format_dataframe(df: DataFrame) -> Value { }) } -/// Convert DataFrame to HTML table -fn dataframe_to_html(df: &DataFrame) -> String { - use ggsql::array_util::value_to_string; - use ggsql::util::escape_html; +/// How many of a result's `height` rows its HTML table shows. +/// +/// A console result is followed by a link to the whole of it in the data +/// explorer, so it shows what R's tibble does: every row up to twice +/// `console_rows`, else the first `console_rows`, which keeps the link in +/// view. Anywhere else the table is all there is, so it shows up to 100. +fn html_rows_shown(height: usize, hints: &RenderHints) -> usize { + let n = hints.console_rows; + match hints.kind { + SessionKind::PositronConsole if height > n.saturating_mul(2) => n, + SessionKind::PositronConsole => height, + _ => height.min(100), + } +} + +/// A cell as a person reads it: `value_to_string`, except that dates are +/// written as dates rather than as the day or millisecond counts Arrow stores. +pub(crate) fn cell_text(col: &arrow::array::ArrayRef, row: usize) -> String { + use arrow::datatypes::DataType; + use arrow::util::display::{ArrayFormatter, FormatOptions}; + match col.data_type() { + DataType::Date32 | DataType::Date64 if !arrow::array::Array::is_null(col, row) => { + ArrayFormatter::try_new(col.as_ref(), &FormatOptions::default()) + .map(|f| f.value(row).to_string()) + .unwrap_or_else(|_| ggsql::array_util::value_to_string(col, row)) + } + _ => ggsql::array_util::value_to_string(col, row), + } +} +/// Convert DataFrame to an HTML table of its first `row_limit` rows +fn dataframe_to_html(df: &DataFrame, row_limit: usize) -> String { let mut html = String::from("\n"); // Header row @@ -283,12 +328,10 @@ fn dataframe_to_html(df: &DataFrame) -> String { } html.push_str("\n\n"); - // Data rows (limit to first 100 for performance) - let row_limit = df.height().min(100); for i in 0..row_limit { html.push_str(""); for col in df.get_columns() { - let value = value_to_string(col, i); + let value = cell_text(col, i); html.push_str(&format!("", escape_html(&value))); } html.push_str("\n"); @@ -308,19 +351,13 @@ fn dataframe_to_html(df: &DataFrame) -> String { /// Convert DataFrame to plain-text summary (shape + column names + first rows). fn dataframe_to_text(df: &ggsql::DataFrame) -> String { - use ggsql::array_util::value_to_string; - let mut s = format!("shape: ({}, {})\n", df.height(), df.width()); let names = df.get_column_names(); s.push_str(&names.join("\t")); s.push('\n'); let row_limit = df.height().min(10); for i in 0..row_limit { - let row: Vec = df - .get_columns() - .iter() - .map(|c| value_to_string(c, i)) - .collect(); + let row: Vec = df.get_columns().iter().map(|c| cell_text(c, i)).collect(); s.push_str(&row.join("\t")); s.push('\n'); } @@ -357,6 +394,17 @@ mod tests { } } + #[test] + fn test_dates_are_written_as_dates() { + let df = ggsql::reader::connection::reader_from_uri("duckdb://memory") + .unwrap() + .execute_sql("SELECT DATE '2024-01-01' AS d, NULL::DATE AS missing") + .unwrap(); + let html = dataframe_to_html(&df, 1); + assert!(html.contains(""), "{html}"); + assert!(dataframe_to_text(&df).contains("2024-01-01")); + } + #[test] fn test_a_console_gets_a_comm_even_without_an_adapter() { // No GPU here, and the console still opens a comm — Positron would @@ -505,6 +553,47 @@ mod tests { ); } + #[test] + fn test_html_rows_shown() { + let console = positron_console(); + // The console shows a short result whole, and a long one's first 10. + assert_eq!(html_rows_shown(20, &console), 20); + assert_eq!(html_rows_shown(21, &console), 10); + assert_eq!(html_rows_shown(5000, &console), 10); + // Anywhere else, up to 100. + assert_eq!(html_rows_shown(50, &positron_notebook()), 50); + assert_eq!(html_rows_shown(5000, &positron_notebook()), 100); + assert_eq!(html_rows_shown(5000, &RenderHints::default()), 100); + } + + #[test] + fn test_console_rows_is_configurable() { + let console = RenderHints { + console_rows: 3, + ..positron_console() + }; + assert_eq!(html_rows_shown(6, &console), 6); + assert_eq!(html_rows_shown(7, &console), 3); + // Only the console's table follows it. + let notebook = RenderHints { + console_rows: 3, + ..positron_notebook() + }; + assert_eq!(html_rows_shown(50, ¬ebook), 50); + } + + #[test] + fn test_console_table_counts_the_rows_it_leaves_out() { + let df = ggsql::reader::connection::reader_from_uri("duckdb://memory") + .unwrap() + .execute_sql("SELECT i FROM range(5000) t(i)") + .unwrap(); + let bundle = format_dataframe(df, &positron_console()); + let html = bundle["data"]["text/html"].as_str().unwrap(); + assert_eq!(html.matches("").count(), 1 + 10 + 1, "{html}"); + assert!(html.contains("... 4990 more rows"), "{html}"); + } + /// A render backend with no GPU, so tests are fast and identical /// everywhere. The SVG path it leaves is the one that always works. fn backend() -> PlotBackend { @@ -514,8 +603,7 @@ mod tests { fn positron_console() -> RenderHints { RenderHints { kind: SessionKind::PositronConsole, - output_width_px: None, - pixel_ratio: None, + ..RenderHints::default() } } @@ -523,7 +611,7 @@ mod tests { RenderHints { kind: SessionKind::PositronNotebook, output_width_px: Some(589), - pixel_ratio: None, + ..RenderHints::default() } } diff --git a/ggsql-jupyter/src/kernel.rs b/ggsql-jupyter/src/kernel.rs index 9eb03f4ae..bc9f9db6e 100644 --- a/ggsql-jupyter/src/kernel.rs +++ b/ggsql-jupyter/src/kernel.rs @@ -5,11 +5,13 @@ use crate::connection; use crate::data_explorer::{DataExplorerState, RpcResponse}; -use crate::display::{format_display_data, Formatted, RenderHints, SessionMode}; +use crate::display::{format_display_data, Formatted, RenderHints, SessionKind, SessionMode}; use crate::executor::{self, ExecutionResult, QueryExecutor}; use crate::message::{ConnectionInfo, JupyterMessage, MessageHeader}; use crate::plot::comm::{PlotMetadata, RenderParams, RpcError}; use crate::plot::{PlotBackend, RenderOutcome, RenderTicket}; +use crate::results::filter::Capabilities; +use crate::results::{methods, ResultStore}; use anyhow::Result; use ggsql::reader::ResolvedPlot; use hmac::{Hmac, Mac}; @@ -65,6 +67,10 @@ pub struct KernelServer { ui_comm_id: Option, connection_comm_id: Option, data_explorer_comms: HashMap, + /// Console query results kept for the results API. + results: ResultStore, + /// `--console-rows`: how many rows of a long result a console shows. + console_rows: usize, } impl KernelServer { @@ -74,6 +80,8 @@ impl KernelServer { reader_uri: &str, session_mode: Option, max_plots: usize, + max_results: usize, + console_rows: usize, ) -> Result { tracing::info!("Initializing kernel server"); @@ -141,6 +149,8 @@ impl KernelServer { ui_comm_id: None, connection_comm_id: None, data_explorer_comms: HashMap::new(), + results: ResultStore::new(max_results)?, + console_rows, }; // Send initial "starting" status on IOPub @@ -358,7 +368,10 @@ impl KernelServer { let content = &parent.content; let code = content["code"].as_str().unwrap_or(""); let silent = content["silent"].as_bool().unwrap_or(false); - let hints = RenderHints::from_request(&parent.header, content, self.session_mode); + let hints = RenderHints { + console_rows: self.console_rows, + ..RenderHints::from_request(&parent.header, content, self.session_mode) + }; // `kind` decides where a plot goes, so log its two inputs: the declared // mode and the session id the heuristic falls back to. A plot in the @@ -408,6 +421,17 @@ impl KernelServer { // the user's. Both stages feed one `Result` and share the error path // below; propagating out of the loop would end the process and take // the session's database with it. + // A console result is kept for the results API, and its key goes in + // the `execute_result`'s metadata so a frontend can offer to explore + // it. A silent cell shows nothing to offer, and DDL has no table. + let mut retained = None; + if let Ok(ExecutionResult::DataFrame(df)) = &executed { + if !silent && hints.kind == SessionKind::PositronConsole && df.width() > 0 { + let title = format!("Query [{}]", self.execution_count); + retained = Some(self.results.insert(df.clone(), title, query_of(code))); + } + } + let result = match executed { Ok(exec_result) => { // A bare connection change renders nothing. @@ -433,10 +457,17 @@ impl KernelServer { // No `output_location`: it routes the output to the Plots // pane as well as the cell, so the plot would arrive twice. Formatted::Bundle(display_data) => { + let mut metadata = display_data["metadata"].clone(); + if let Some(key) = &retained { + if !metadata.is_object() { + metadata = json!({}); + } + metadata["ggsql"] = json!({ "result": key }); + } let content = json!({ "execution_count": self.execution_count, "data": display_data["data"], - "metadata": display_data["metadata"] + "metadata": metadata }); self.send_iopub("execute_result", content, parent).await?; } @@ -712,16 +743,43 @@ impl KernelServer { if rpc_id.is_null() { self.handle_ui_notification(method, &data["params"]); } else { + let reply = match method { + // A frontend extension's call into the kernel + // (`session.callMethod`). The results API is the only + // thing answered this way. + "call_method" => { + let name = data["params"]["method"].as_str().unwrap_or(""); + let args = &data["params"]["params"]; + let answer = if methods::handles(name) { + let session = methods::Session { + reader: self.executor.reader(), + caps: Capabilities { + regex: self.executor.reader_uri().starts_with("duckdb://"), + }, + }; + methods::call(&mut self.results, &session, name, args) + } else { + Err(format!("Unknown method {name:?}")) + }; + match answer { + Ok(result) => { + json!({"jsonrpc": "2.0", "id": rpc_id, "result": result}) + } + Err(message) => { + tracing::warn!("{} failed: {}", name, message); + json!({ + "jsonrpc": "2.0", + "id": rpc_id, + "error": RpcError::Internal(message).to_json() + }) + } + } + } + _ => json!({"jsonrpc": "2.0", "id": rpc_id, "result": null}), + }; self.send_shell_reply( "comm_msg", - json!({ - "comm_id": comm_id, - "data": { - "jsonrpc": "2.0", - "id": rpc_id, - "result": null - } - }), + json!({ "comm_id": comm_id, "data": reply }), parent, identities, ) @@ -1620,6 +1678,15 @@ impl KernelServer { } } +/// A cell's SQL without its leading meta-commands, which are not SQL. +fn query_of(code: &str) -> String { + let mut rest = code; + while let Some((_, after)) = executor::take_leading_meta(rest) { + rest = after; + } + rest.trim().to_string() +} + /// Check if ggsql code is complete (balanced brackets, not in a string) /// If the code contains VISUALISE, it must also contain at least one DRAW layer fn is_code_complete(code: &str) -> bool { diff --git a/ggsql-jupyter/src/lib.rs b/ggsql-jupyter/src/lib.rs index 7b06b0960..d834df295 100644 --- a/ggsql-jupyter/src/lib.rs +++ b/ggsql-jupyter/src/lib.rs @@ -8,6 +8,7 @@ pub mod display; pub mod executor; pub mod message; pub mod plot; +pub mod results; // Re-export commonly used types pub use display::format_display_data; pub use executor::{ExecutionResult, QueryExecutor}; diff --git a/ggsql-jupyter/src/main.rs b/ggsql-jupyter/src/main.rs index 2cd08240a..a99f1d9d7 100644 --- a/ggsql-jupyter/src/main.rs +++ b/ggsql-jupyter/src/main.rs @@ -10,6 +10,7 @@ mod executor; mod kernel; mod message; mod plot; +mod results; use anyhow::{Context, Result}; use clap::Parser; use display::SessionMode; @@ -48,6 +49,29 @@ struct Args { #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u32).range(1..))] max_plots: u32, + /// How many query results to keep available for the data explorer. + /// + /// In a Positron console each tabular result is kept for the results API, + /// so a frontend can open it in a data explorer later. This bounds the + /// results not opened yet, dropping the oldest; opened results are bounded + /// by `GGSQL_RESULTS_MAX_BYTES` instead (default 1 GB). + #[arg(long, default_value_t = 16, value_parser = clap::value_parser!(u32).range(1..))] + max_results: u32, + + /// How many rows of a query result a Positron console shows. + /// + /// A result of up to twice this many rows is shown whole; a longer one + /// shows this many, then a count of the rest. Also read from the + /// environment, which is how the ggsql extension sets it: a kernel that + /// predates the option ignores a variable but would refuse a flag. + #[arg( + long, + env = "GGSQL_CONSOLE_ROWS", + default_value_t = 10, + value_parser = clap::value_parser!(u32).range(1..) + )] + console_rows: u32, + /// Install the kernel spec #[arg(long)] install: bool, @@ -100,6 +124,8 @@ async fn main() -> Result<()> { &args.reader, args.session_mode, args.max_plots as usize, + args.max_results as usize, + args.console_rows as usize, ) .await?; diff --git a/ggsql-jupyter/src/results/export.rs b/ggsql-jupyter/src/results/export.rs new file mode 100644 index 000000000..9505742d5 --- /dev/null +++ b/ggsql-jupyter/src/results/export.rs @@ -0,0 +1,108 @@ +//! A data explorer selection as CSV, TSV or HTML, for the clipboard. + +use ggsql::util::escape_html; + +/// Format `rows` for export. A `header` is written first when given; a null +/// cell is empty. +pub fn format( + header: Option<&[String]>, + rows: &[Vec], + format: &str, +) -> Result { + let all = header + .into_iter() + .map(<[String]>::to_vec) + .chain(rows.iter().cloned()); + match format { + "csv" => Ok(delimited(all, ",", csv_field)), + "tsv" => Ok(delimited(all, "\t", tsv_field)), + "html" => Ok(html(header, rows)), + other => Err(format!("Unsupported export format {other:?}")), + } +} + +fn delimited( + rows: impl Iterator>, + sep: &str, + field: fn(&str) -> String, +) -> String { + let lines: Vec = rows + .map(|row| row.iter().map(|v| field(v)).collect::>().join(sep)) + .collect(); + lines.join("\n") +} + +/// Quoted only when it has to be, with embedded quotes doubled (RFC 4180). +fn csv_field(v: &str) -> String { + if v.contains([',', '"', '\n', '\r']) { + format!("\"{}\"", v.replace('"', "\"\"")) + } else { + v.to_string() + } +} + +/// TSV has no quoting, so the characters that would break a row become spaces. +fn tsv_field(v: &str) -> String { + v.replace(['\t', '\n', '\r'], " ") +} + +fn html(header: Option<&[String]>, rows: &[Vec]) -> String { + let mut out = String::from("
{}
2024-01-01
\n"); + if let Some(header) = header { + out.push_str(""); + for h in header { + out.push_str(&format!("", escape_html(h))); + } + out.push_str("\n"); + } + out.push_str("\n"); + for row in rows { + out.push_str(""); + for v in row { + out.push_str(&format!("", escape_html(v))); + } + out.push_str("\n"); + } + out.push_str("\n
{}
{}
"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rows() -> Vec> { + vec![ + vec!["1".into(), "a, \"b\"".into()], + vec!["2".into(), "tab\there".into()], + ] + } + + #[test] + fn test_csv_quotes_only_what_needs_it() { + let header = ["n".to_string(), "s".to_string()]; + assert_eq!( + format(Some(&header), &rows(), "csv").unwrap(), + "n,s\n1,\"a, \"\"b\"\"\"\n2,tab\there" + ); + } + + #[test] + fn test_tsv_flattens_tabs() { + assert_eq!( + format(None, &rows(), "tsv").unwrap(), + "1\ta, \"b\"\n2\ttab here" + ); + } + + #[test] + fn test_html_escapes() { + let out = format(None, &[vec!["".into()]], "html").unwrap(); + assert!(out.contains("<b>"), "{out}"); + } + + #[test] + fn test_unknown_format_is_an_error() { + assert!(format(None, &rows(), "xlsx").is_err()); + } +} diff --git a/ggsql-jupyter/src/results/filter.rs b/ggsql-jupyter/src/results/filter.rs new file mode 100644 index 000000000..ae2ea05e9 --- /dev/null +++ b/ggsql-jupyter/src/results/filter.rs @@ -0,0 +1,472 @@ +//! Row filters and sort keys over a result's columns, and the SQL for them. +//! +//! Everything here builds from the request and the result's schema, with no +//! database, so it is tested as plain string assembly. + +use super::Column; +use ggsql::naming::{quote_ident, quote_literal}; +use ggsql::reader::SqlDialect; +use serde::{Deserialize, Serialize}; + +/// What a database can do beyond the SQL every backend shares. +#[derive(Debug, Clone, Copy)] +pub struct Capabilities { + /// `regexp_matches`, for regular-expression text filters. DuckDB only. + pub regex: bool, +} + +/// A comparison operator. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum CompareOp { + #[serde(rename = "=")] + Eq, + #[serde(rename = "!=")] + NotEq, + #[serde(rename = "<")] + Lt, + #[serde(rename = "<=")] + LtEq, + #[serde(rename = ">")] + Gt, + #[serde(rename = ">=")] + GtEq, +} + +/// How a text filter matches. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TextMatch { + Contains, + NotContains, + StartsWith, + EndsWith, + Regex, +} + +/// One row filter. `column` is an index into the result's columns, and every +/// value is a string, checked against the column's type before it reaches SQL. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Filter { + Compare { + column: usize, + op: CompareOp, + value: String, + }, + Between { + column: usize, + low: String, + high: String, + #[serde(default)] + negate: bool, + }, + IsNull { + column: usize, + #[serde(default)] + negate: bool, + }, + IsEmpty { + column: usize, + #[serde(default)] + negate: bool, + }, + IsTrue { + column: usize, + }, + IsFalse { + column: usize, + }, + Text { + column: usize, + #[serde(rename = "match")] + how: TextMatch, + term: String, + #[serde(default)] + case_sensitive: bool, + }, + In { + column: usize, + values: Vec, + #[serde(default)] + negate: bool, + }, +} + +impl Filter { + fn column(&self) -> usize { + match self { + Filter::Compare { column, .. } + | Filter::Between { column, .. } + | Filter::IsNull { column, .. } + | Filter::IsEmpty { column, .. } + | Filter::IsTrue { column } + | Filter::IsFalse { column } + | Filter::Text { column, .. } + | Filter::In { column, .. } => *column, + } + } +} + +/// One sort key. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SortKey { + pub column: usize, + #[serde(default = "ascending")] + pub ascending: bool, +} + +fn ascending() -> bool { + true +} + +fn column_of(columns: &[Column], index: usize) -> Result<&Column, String> { + columns + .get(index) + .ok_or_else(|| format!("There is no column {index}")) +} + +/// The SQL predicate for one row filter. +pub fn predicate( + filter: &Filter, + columns: &[Column], + dialect: &dyn SqlDialect, + caps: Capabilities, +) -> Result { + let column = column_of(columns, filter.column())?; + let c = quote_ident(&column.name); + let lit = |v: &str| literal(v, &column.kind, dialect); + + match filter { + Filter::Compare { op, value, .. } => { + let op = match op { + CompareOp::Eq => "=", + CompareOp::NotEq => "<>", + CompareOp::Lt => "<", + CompareOp::LtEq => "<=", + CompareOp::Gt => ">", + CompareOp::GtEq => ">=", + }; + Ok(format!("{c} {op} {}", lit(value)?)) + } + Filter::Between { + low, high, negate, .. + } => { + let between = format!("{c} BETWEEN {} AND {}", lit(low)?, lit(high)?); + Ok(if *negate { + format!("NOT ({between})") + } else { + between + }) + } + Filter::IsNull { negate: false, .. } => Ok(format!("{c} IS NULL")), + Filter::IsNull { negate: true, .. } => Ok(format!("{c} IS NOT NULL")), + Filter::IsEmpty { negate: false, .. } => Ok(format!("{c} = ''")), + Filter::IsEmpty { negate: true, .. } => Ok(format!("{c} <> ''")), + Filter::IsTrue { .. } => Ok(format!("{c} = {}", dialect.sql_boolean_literal(true))), + Filter::IsFalse { .. } => Ok(format!("{c} = {}", dialect.sql_boolean_literal(false))), + Filter::Text { + how, + term, + case_sensitive, + .. + } => text(&c, *how, term, *case_sensitive, caps), + Filter::In { values, negate, .. } => { + if values.is_empty() { + // Nothing selected: keep nothing, or exclude nothing. + return Ok(if *negate { "1 = 1" } else { "1 = 0" }.to_string()); + } + let list = values + .iter() + .map(|v| lit(v)) + .collect::, _>>()?; + let not = if *negate { "NOT " } else { "" }; + Ok(format!("{c} {not}IN ({})", list.join(", "))) + } + } +} + +/// A filter value as a SQL literal of its column's kind. +fn literal(value: &str, kind: &str, dialect: &dyn SqlDialect) -> Result { + match kind { + "integer" | "floating" => { + let v = value.trim(); + // Only a finite number passes, so nothing else reaches the SQL + // unquoted. + match v.parse::() { + Ok(n) if n.is_finite() => Ok(v.to_string()), + _ => Err(format!("{value:?} is not a number")), + } + } + "boolean" => match value.trim().to_ascii_lowercase().as_str() { + "true" | "1" => Ok(dialect.sql_boolean_literal(true)), + "false" | "0" => Ok(dialect.sql_boolean_literal(false)), + _ => Err(format!("{value:?} is not true or false")), + }, + // Dates and times too: every backend compares a string literal against + // a temporal column by casting it. + _ => Ok(quote_literal(value)), + } +} + +/// The predicate for a text filter. +fn text( + c: &str, + how: TextMatch, + term: &str, + case_sensitive: bool, + caps: Capabilities, +) -> Result { + if how == TextMatch::Regex { + if !caps.regex { + return Err("This database does not support regular-expression search".into()); + } + let flags = if case_sensitive { "" } else { ", 'i'" }; + return Ok(format!( + "regexp_matches({c}, {}{flags})", + quote_literal(term) + )); + } + + let escaped = escape_like(term); + let (pattern, negate) = match how { + TextMatch::Contains => (format!("%{escaped}%"), false), + TextMatch::NotContains => (format!("%{escaped}%"), true), + TextMatch::StartsWith => (format!("{escaped}%"), false), + TextMatch::EndsWith => (format!("%{escaped}"), false), + TextMatch::Regex => unreachable!("handled above"), + }; + let (lhs, rhs) = if case_sensitive { + (c.to_string(), quote_literal(&pattern)) + } else { + ( + format!("LOWER({c})"), + format!("LOWER({})", quote_literal(&pattern)), + ) + }; + let not = if negate { "NOT " } else { "" }; + Ok(format!("{lhs} {not}LIKE {rhs} ESCAPE '\\'")) +} + +/// Escape `LIKE`'s wildcards, so a search term matches literally. +fn escape_like(term: &str) -> String { + term.replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") +} + +/// The sort keys as `ORDER BY` terms, without the keyword. +pub fn order_terms(keys: &[SortKey], columns: &[Column]) -> Result, String> { + keys.iter() + .map(|key| { + let column = column_of(columns, key.column)?; + let direction = if key.ascending { "ASC" } else { "DESC" }; + Ok(format!("{} {direction}", quote_ident(&column.name))) + }) + .collect() +} + +/// SQL reproducing a view of a result: `from` filtered and sorted. +/// +/// A filter that cannot be expressed in `dialect` is left out with a comment +/// saying so, rather than failing the whole conversion. +pub fn to_sql( + filters: &[Filter], + sort: &[SortKey], + from: &str, + columns: &[Column], + dialect: &dyn SqlDialect, + caps: Capabilities, +) -> Vec { + let mut lines = Vec::new(); + let mut predicates = Vec::new(); + for filter in filters { + match predicate(filter, columns, dialect, caps) { + Ok(p) => predicates.push(p), + Err(e) => lines.push(format!("-- Filter left out: {e}")), + } + } + + lines.push("SELECT *".to_string()); + lines.extend(format!("FROM {from}").lines().map(str::to_string)); + for (i, p) in predicates.iter().enumerate() { + let keyword = if i == 0 { "WHERE" } else { " AND" }; + lines.push(format!("{keyword} {p}")); + } + match order_terms(sort, columns) { + Ok(terms) if !terms.is_empty() => lines.push(format!("ORDER BY {}", terms.join(", "))), + Ok(_) => {} + Err(e) => lines.push(format!("-- Sort left out: {e}")), + } + lines +} + +/// A query as a derived table, for wrapping in a view's filters. +pub fn subquery(query: &str) -> String { + let query = query.trim().trim_end_matches(';').trim_end(); + let body: Vec = query.lines().map(|l| format!(" {l}")).collect(); + format!("(\n{}\n) AS {}", body.join("\n"), quote_ident("result")) +} + +#[cfg(test)] +mod tests { + use super::*; + use ggsql::reader::AnsiDialect; + use serde_json::json; + + const CAPS: Capabilities = Capabilities { regex: true }; + + fn columns() -> Vec { + [("n", "integer"), ("name", "string"), ("ok", "boolean")] + .iter() + .map(|(name, kind)| Column { + name: name.to_string(), + type_name: String::new(), + kind: kind.to_string(), + }) + .collect() + } + + fn sql(f: Value) -> Result { + let filter: Filter = serde_json::from_value(f).unwrap(); + predicate(&filter, &columns(), &AnsiDialect, CAPS) + } + + use serde_json::Value; + + #[test] + fn test_compare_maps_not_equal_and_checks_numbers() { + assert_eq!( + sql(json!({"type": "compare", "column": 0, "op": "!=", "value": "3"})).unwrap(), + "\"n\" <> 3" + ); + assert!( + sql(json!({"type": "compare", "column": 0, "op": "=", "value": "1; DROP"})).is_err() + ); + assert!(sql(json!({"type": "compare", "column": 0, "op": "=", "value": "inf"})).is_err()); + } + + #[test] + fn test_string_values_are_quoted() { + assert_eq!( + sql(json!({"type": "compare", "column": 1, "op": "=", "value": "O'Brien"})).unwrap(), + "\"name\" = 'O''Brien'" + ); + } + + #[test] + fn test_negated_between() { + assert_eq!( + sql(json!({"type": "between", "column": 0, "low": "1", "high": "5", "negate": true})) + .unwrap(), + "NOT (\"n\" BETWEEN 1 AND 5)" + ); + } + + #[test] + fn test_text_escapes_wildcards_and_folds_case() { + assert_eq!( + sql(json!({"type": "text", "column": 1, "match": "contains", "term": "50%_"})).unwrap(), + "LOWER(\"name\") LIKE LOWER('%50\\%\\_%') ESCAPE '\\'" + ); + assert_eq!( + sql( + json!({"type": "text", "column": 1, "match": "starts_with", "term": "a", + "case_sensitive": true}) + ) + .unwrap(), + "\"name\" LIKE 'a%' ESCAPE '\\'" + ); + } + + #[test] + fn test_regex_needs_the_capability() { + let f: Filter = serde_json::from_value( + json!({"type": "text", "column": 1, "match": "regex", "term": "^a"}), + ) + .unwrap(); + assert_eq!( + predicate(&f, &columns(), &AnsiDialect, CAPS).unwrap(), + "regexp_matches(\"name\", '^a', 'i')" + ); + assert!(predicate(&f, &columns(), &AnsiDialect, Capabilities { regex: false }).is_err()); + } + + #[test] + fn test_in() { + assert_eq!( + sql(json!({"type": "in", "column": 1, "values": ["a", "b"], "negate": true})).unwrap(), + "\"name\" NOT IN ('a', 'b')" + ); + assert_eq!( + sql(json!({"type": "in", "column": 1, "values": []})).unwrap(), + "1 = 0" + ); + } + + #[test] + fn test_booleans_use_the_dialect() { + assert_eq!( + sql(json!({"type": "is_true", "column": 2})).unwrap(), + format!("\"ok\" = {}", AnsiDialect.sql_boolean_literal(true)) + ); + assert!(sql(json!({"type": "compare", "column": 2, "op": "=", "value": "maybe"})).is_err()); + } + + #[test] + fn test_an_unknown_column_is_an_error() { + assert!(sql(json!({"type": "is_null", "column": 9})).is_err()); + assert!(order_terms( + &[SortKey { + column: 9, + ascending: true + }], + &columns() + ) + .is_err()); + } + + #[test] + fn test_to_sql_wraps_the_query() { + let filters: Vec = serde_json::from_value(json!([ + {"type": "compare", "column": 0, "op": ">", "value": "2"}, + {"type": "is_null", "column": 1} + ])) + .unwrap(); + let sort = [SortKey { + column: 0, + ascending: true, + }]; + let from = subquery("SELECT *\nFROM t;"); + assert_eq!( + to_sql(&filters, &sort, &from, &columns(), &AnsiDialect, CAPS), + vec![ + "SELECT *", + "FROM (", + " SELECT *", + " FROM t", + ") AS \"result\"", + "WHERE \"n\" > 2", + " AND \"name\" IS NULL", + "ORDER BY \"n\" ASC", + ] + ); + } + + #[test] + fn test_to_sql_leaves_out_what_it_cannot_express() { + let filters: Vec = serde_json::from_value(json!([ + {"type": "text", "column": 1, "match": "regex", "term": "x", "case_sensitive": true} + ])) + .unwrap(); + let lines = to_sql( + &filters, + &[], + "\"t\"", + &columns(), + &AnsiDialect, + Capabilities { regex: false }, + ); + assert!(lines[0].starts_with("-- Filter left out"), "{lines:?}"); + assert_eq!(&lines[1..], ["SELECT *", "FROM \"t\""]); + } +} diff --git a/ggsql-jupyter/src/results/methods.rs b/ggsql-jupyter/src/results/methods.rs new file mode 100644 index 000000000..d961e2aa9 --- /dev/null +++ b/ggsql-jupyter/src/results/methods.rs @@ -0,0 +1,246 @@ +//! The results API as named JSON calls. +//! +//! | Method | Params | Result | +//! | --- | --- | --- | +//! | `result_open` | `{key}` | `{view, key, title, query, columns: [{name, type_name, kind}], num_rows}` | +//! | `result_set_filters` | `{view, filters: [Filter]}` | `{num_rows, errors: [string \| null]}` | +//! | `result_set_sort` | `{view, keys: [{column, ascending}]}` | `{}` | +//! | `result_page` | `{view, rows: {first, last} \| {indices}, columns: [index]}` | `{columns: [[string \| null]]}` | +//! | `result_profile` | `{view, column, profiles: {null_count?, summary?, histogram?: {bins, quantiles}, frequency?: {limit}}}` | `{null_count?, summary?, histogram?, frequency?}` | +//! | `result_export` | `{view, rows?, columns, format: csv \| tsv \| html}` | `{data}` | +//! | `result_sql` | `{view}` | `{sql: [line]}`, in the session's dialect | +//! | `result_close` | `{view}` | `{}` | +//! +//! `num_rows` after filtering counts the rows that pass. A filter that cannot +//! be applied is left out and its message returned at its index in `errors`. + +use super::filter::{Capabilities, Filter, SortKey}; +use super::view::{ProfileRequest, Rows}; +use super::ResultStore; +use ggsql::reader::Reader; +use serde::de::DeserializeOwned; +use serde::Deserialize; +use serde_json::{json, Value}; + +/// The method names this module answers. +pub fn handles(method: &str) -> bool { + method.starts_with("result_") +} + +/// The session's database, which `result_sql` writes SQL for. +pub struct Session<'a> { + pub reader: &'a dyn Reader, + pub caps: Capabilities, +} + +/// The results cache is DuckDB, which has `regexp_matches`. +const CACHE_CAPS: Capabilities = Capabilities { regex: true }; + +#[derive(Deserialize)] +struct Open { + key: String, +} + +#[derive(Deserialize)] +struct ViewOnly { + view: String, +} + +#[derive(Deserialize)] +struct SetFilters { + view: String, + filters: Vec, +} + +#[derive(Deserialize)] +struct SetSort { + view: String, + keys: Vec, +} + +#[derive(Deserialize)] +struct Page { + view: String, + rows: Rows, + columns: Vec, +} + +#[derive(Deserialize)] +struct Profile { + view: String, + column: usize, + profiles: ProfileRequest, +} + +#[derive(Deserialize)] +struct Export { + view: String, + rows: Option, + columns: Vec, + format: String, +} + +/// Params arrive as the arguments of a call: an array whose first element is +/// the object, or the object itself. +fn parse(params: &Value) -> Result { + let object = match params { + Value::Array(args) => args.first().cloned().unwrap_or(Value::Null), + other => other.clone(), + }; + serde_json::from_value(object).map_err(|e| format!("Invalid parameters: {e}")) +} + +/// Answer one call. +pub fn call( + store: &mut ResultStore, + session: &Session, + method: &str, + params: &Value, +) -> Result { + let err = |e: anyhow::Error| e.to_string(); + match method { + "result_open" => { + let p: Open = parse(params)?; + let opened = store.open(&p.key).map_err(err)?; + serde_json::to_value(opened).map_err(|e| e.to_string()) + } + "result_set_filters" => { + let p: SetFilters = parse(params)?; + let (view, cache) = store.view_mut(&p.view).map_err(err)?; + let errors = view.set_filters(p.filters, cache.backend(), CACHE_CAPS)?; + Ok(json!({ "num_rows": view.selected_rows, "errors": errors })) + } + "result_set_sort" => { + let p: SetSort = parse(params)?; + let (view, cache) = store.view_mut(&p.view).map_err(err)?; + view.set_sort(p.keys, cache.backend())?; + Ok(json!({})) + } + "result_page" => { + let p: Page = parse(params)?; + let (view, cache) = store.view_mut(&p.view).map_err(err)?; + let columns = view.page(cache.backend(), &p.rows, &p.columns)?; + Ok(json!({ "columns": columns })) + } + "result_profile" => { + let p: Profile = parse(params)?; + let (view, cache) = store.view_mut(&p.view).map_err(err)?; + view.profile(cache.backend(), p.column, &p.profiles) + } + "result_export" => { + let p: Export = parse(params)?; + let (view, cache) = store.view_mut(&p.view).map_err(err)?; + let data = view.export(cache.backend(), p.rows.as_ref(), &p.columns, &p.format)?; + Ok(json!({ "data": data })) + } + "result_sql" => { + let p: ViewOnly = parse(params)?; + let query = store.query_of(&p.view).map_err(err)?.to_string(); + let (view, _) = store.view_mut(&p.view).map_err(err)?; + let sql = view.to_sql(&query, session.reader.dialect(), session.caps); + Ok(json!({ "sql": sql })) + } + "result_close" => { + let p: ViewOnly = parse(params)?; + store.close(&p.view).map_err(err)?; + Ok(json!({})) + } + other => Err(format!("Unknown method {other:?}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ggsql::reader::connection::reader_from_uri; + + fn setup() -> (ResultStore, Box, String) { + let session = reader_from_uri("duckdb://memory").unwrap(); + let df = session + .execute_sql("SELECT i AS n, 'x' || i AS s FROM range(5) t(i)") + .unwrap(); + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(df, "Query [1]".into(), "SELECT * FROM t;".into()); + (store, session, key) + } + + #[test] + fn test_a_round_trip_through_the_calls() { + let (mut store, reader, key) = setup(); + let session = Session { + reader: &*reader, + caps: Capabilities { regex: true }, + }; + let mut call = |m: &str, p: Value| call(&mut store, &session, m, &p); + + // Positional arguments, as `callMethod` sends them. + let opened = call("result_open", json!([{ "key": key }])).unwrap(); + assert_eq!(opened["num_rows"], 5); + assert_eq!(opened["title"], "Query [1]"); + let view = opened["view"].as_str().unwrap().to_string(); + + let r = call( + "result_set_filters", + json!({"view": view, "filters": [{"type": "compare", "column": 0, "op": ">", "value": "1"}]}), + ) + .unwrap(); + assert_eq!(r, json!({"num_rows": 3, "errors": [null]})); + + call( + "result_set_sort", + json!({"view": view, "keys": [{"column": 0, "ascending": false}]}), + ) + .unwrap(); + let page = call( + "result_page", + json!({"view": view, "rows": {"first": 0, "last": 1}, "columns": [1]}), + ) + .unwrap(); + assert_eq!(page, json!({"columns": [["x4", "x3"]]})); + + let sql = call("result_sql", json!({ "view": view })).unwrap(); + let sql: Vec<&str> = sql["sql"] + .as_array() + .unwrap() + .iter() + .map(|l| l.as_str().unwrap()) + .collect(); + assert_eq!( + sql, + [ + "SELECT *", + "FROM (", + " SELECT * FROM t", + ") AS \"result\"", + "WHERE \"n\" > 1", + "ORDER BY \"n\" DESC" + ] + ); + + assert_eq!( + call("result_close", json!({ "view": view })).unwrap(), + json!({}) + ); + assert!(call( + "result_page", + json!({"view": view, "rows": {"first": 0, "last": 0}, "columns": [0]}) + ) + .unwrap_err() + .contains("no view")); + } + + #[test] + fn test_bad_params_and_unknown_methods_are_errors() { + let (mut store, reader, _) = setup(); + let session = Session { + reader: &*reader, + caps: Capabilities { regex: false }, + }; + assert!(call(&mut store, &session, "result_open", &json!({})) + .unwrap_err() + .contains("Invalid parameters")); + assert!(call(&mut store, &session, "result_nope", &json!({})).is_err()); + assert!(handles("result_page")); + assert!(!handles("open_editor")); + } +} diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs new file mode 100644 index 000000000..5e189f6de --- /dev/null +++ b/ggsql-jupyter/src/results/mod.rs @@ -0,0 +1,729 @@ +//! Query results kept so a frontend can explore them after the fact. +//! +//! A frontend-agnostic API over recent results, one page at a time: open a +//! result, filter and sort it, read pages, profile columns, export a selection, +//! and get SQL reproducing the view. [`methods`] exposes it as named JSON +//! calls. Nothing here knows about any particular frontend. +//! +//! A result is held as its Arrow data until it is first opened, and is then +//! copied into a dedicated in-memory cache ([`CacheTables`], the same +//! machinery as ggsql's caching reader) where every filter, sort, page and +//! profile is SQL. The cache is the results API's own: the session's reader is +//! never wrapped, so nothing changes about the dialect the user's queries run +//! in. + +mod export; +pub mod filter; +pub mod methods; +pub mod view; + +use anyhow::{anyhow, Result}; +use ggsql::naming::quote_ident; +use ggsql::reader::cache::{parse_human_bytes, CacheConfig, CacheTables}; +use ggsql::DataFrame; +use serde::Serialize; +use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; +use view::ResultView; + +/// The column holding each row's position in the query's output, added when a +/// result is copied into the cache. Ties in a sort break on it, so paging is +/// stable and clearing the sort restores the query's own order. +pub const ROW: &str = "__ggsql_row__"; + +/// The results cache's default byte budget. +const DEFAULT_MAX_BYTES: u64 = 1024 * 1024 * 1024; + +/// A result's column. +#[derive(Debug, Clone, Serialize)] +pub struct Column { + pub name: String, + /// The cache backend's type name, e.g. `BIGINT`. + pub type_name: String, + /// A frontend-neutral kind: `integer`, `floating`, `boolean`, `string`, + /// `date`, `datetime`, `time` or `unknown`. + pub kind: String, +} + +/// A result not yet opened, held as its Arrow data. +struct Pending { + key: String, + title: String, + query: String, + df: DataFrame, +} + +/// A result copied into the cache. +struct Opened { + title: String, + query: String, + table: String, + columns: Vec, + num_rows: usize, +} + +/// What `open` reports about a view. +#[derive(Debug, Serialize)] +pub struct OpenedView { + pub view: String, + pub key: String, + pub title: String, + pub query: String, + pub columns: Vec, + pub num_rows: usize, +} + +/// Recent query results and the open views of them. +pub struct ResultStore { + /// The results cache: opened results, pinned while a view is open. + cache: CacheTables, + /// Results not opened yet, oldest first, bounded by `max_pending`. + pending: VecDeque, + max_pending: usize, + opened: HashMap, + views: HashMap, +} + +impl ResultStore { + /// A store keeping up to `max_pending` unopened results. Opened results + /// are bounded by the cache's byte budget, `GGSQL_RESULTS_MAX_BYTES` + /// (default 1 GB). + pub fn new(max_pending: usize) -> Result { + let max_bytes = std::env::var("GGSQL_RESULTS_MAX_BYTES") + .ok() + .and_then(|v| parse_human_bytes(&v)) + .unwrap_or(DEFAULT_MAX_BYTES); + Self::with_max_bytes(max_pending, max_bytes) + } + + /// A store with an explicit byte budget for opened results. + pub fn with_max_bytes(max_pending: usize, max_bytes: u64) -> Result { + let config = CacheConfig { + enabled: true, + ttl_secs: u64::MAX, + max_bytes, + }; + let cache = CacheTables::in_memory(config)?; + // An in-memory DuckDB may otherwise spill to temporary files under + // memory pressure; results must never be written to disk. + cache.backend().execute_sql("SET temp_directory = ''")?; + Ok(Self { + cache, + pending: VecDeque::new(), + max_pending: max_pending.max(1), + opened: HashMap::new(), + views: HashMap::new(), + }) + } + + /// Keep `df`, produced by `query`, under a fresh key, which is returned. + pub fn insert(&mut self, df: DataFrame, title: String, query: String) -> String { + // Unguessable, so a stale link from a previous kernel process can never + // name a result of this one. + let key = uuid::Uuid::new_v4().simple().to_string(); + self.pending.push_back(Pending { + key: key.clone(), + title, + query, + df, + }); + while self.pending.len() > self.max_pending { + self.pending.pop_front(); + } + key + } + + fn table_name(key: &str) -> String { + format!("__ggsql_result_{key}__") + } + + /// Open a new view of the result `key`, copying it into the cache the + /// first time. The result stays pinned while any view of it is open. + pub fn open(&mut self, key: &str) -> Result { + if let Some(i) = self.pending.iter().position(|p| p.key == key) { + let Pending { + key, + title, + query, + df, + } = self.pending.remove(i).expect("position is in range"); + match self.copy_into_cache(&key, title.clone(), query.clone(), df.clone()) { + Ok(opened) => { + self.opened.insert(key, opened); + } + Err(e) => { + // Kept, so the error says what went wrong each time + // rather than that the result is gone. + self.pending.insert( + i, + Pending { + key, + title, + query, + df, + }, + ); + return Err(e); + } + } + } else if self.opened.contains_key(key) { + if self.cache.lookup(key)?.is_none() { + self.opened.remove(key); + return Err(gone()); + } + self.cache.set_pinned(key, true)?; + } else { + return Err(gone()); + } + + let opened = &self.opened[key]; + let id = uuid::Uuid::new_v4().simple().to_string(); + let view = ResultView::new( + key, + &id, + &opened.table, + opened.columns.clone(), + opened.num_rows, + ); + self.views.insert(id.clone(), view); + Ok(OpenedView { + view: id, + key: key.to_string(), + title: opened.title.clone(), + query: opened.query.clone(), + columns: opened.columns.clone(), + num_rows: opened.num_rows, + }) + } + + fn copy_into_cache( + &self, + key: &str, + title: String, + query: String, + df: DataFrame, + ) -> Result { + let num_rows = df.height(); + let df = unique_names(df)?; + let rows = arrow::array::UInt64Array::from_iter_values(0..num_rows as u64); + let df = df.with_column(ROW, Arc::new(rows))?; + let table = Self::table_name(key); + // Pinned from the start, so the eviction its own insertion triggers + // can't take it. + self.cache.put(key, &query, &table, df, true)?; + + let described = self + .cache + .backend() + .execute_sql(&format!("DESCRIBE {}", quote_ident(&table)))?; + let names = ggsql::array_util::as_str(described.column("column_name")?)?; + let types = ggsql::array_util::as_str(described.column("column_type")?)?; + let columns = (0..described.height()) + .filter(|&i| names.value(i) != ROW) + .map(|i| Column { + name: names.value(i).to_string(), + type_name: types.value(i).to_string(), + kind: kind_of(types.value(i)).to_string(), + }) + .collect(); + Ok(Opened { + title, + query, + table, + columns, + num_rows, + }) + } + + /// Close a view, dropping what it keeps in the cache. Its result is + /// unpinned once no view of it is open, and is then evicted when the + /// cache needs the room. + pub fn close(&mut self, view: &str) -> Result<()> { + let mut v = self + .views + .remove(view) + .ok_or_else(|| anyhow!("There is no view {view:?}"))?; + v.close(self.cache.backend()); + if !self.views.values().any(|other| other.key == v.key) { + self.cache.set_pinned(&v.key, false)?; + } + Ok(()) + } + + /// A view and the cache it reads from. + pub fn view_mut(&mut self, view: &str) -> Result<(&mut ResultView, &CacheTables)> { + let v = self + .views + .get_mut(view) + .ok_or_else(|| anyhow!("There is no view {view:?}"))?; + Ok((v, &self.cache)) + } + + /// The query a view's result came from. + pub fn query_of(&self, view: &str) -> Result<&str> { + let v = self + .views + .get(view) + .ok_or_else(|| anyhow!("There is no view {view:?}"))?; + Ok(&self.opened[&v.key].query) + } +} + +/// `df` with each repeated column name made unique, as DuckDB does: the second +/// `a` becomes `a_1`, the third `a_2`, skipping names already taken. A query +/// can return repeated names, but a table cannot hold them. +fn unique_names(df: DataFrame) -> Result { + let names = df.get_column_names(); + let mut taken: std::collections::HashSet = names.iter().cloned().collect(); + let mut seen = std::collections::HashSet::new(); + let mut changed = false; + let mut columns = Vec::with_capacity(names.len()); + for (name, array) in names.iter().zip(df.get_columns()) { + let mut unique = name.clone(); + if !seen.insert(name.clone()) { + let mut n = 1; + while taken.contains(&format!("{name}_{n}")) { + n += 1; + } + unique = format!("{name}_{n}"); + taken.insert(unique.clone()); + changed = true; + } + columns.push((unique, array.clone())); + } + if !changed { + return Ok(df); + } + Ok(DataFrame::new(columns)?) +} + +fn gone() -> anyhow::Error { + anyhow!("This query result is no longer available. Re-run the query to explore it.") +} + +/// A cache type name's neutral kind. +pub fn kind_of(type_name: &str) -> &'static str { + let upper = type_name.to_uppercase(); + let upper = upper.as_str(); + // Nested types first: `INTEGER[]` and `STRUCT(a INTEGER)` name a scalar + // type inside them, but are not numbers. + let nested = upper.ends_with(']') + || ["STRUCT", "MAP", "UNION", "LIST"] + .iter() + .any(|t| upper.starts_with(t)); + if nested || upper.contains("INTERVAL") { + "unknown" + } else if upper.contains("INT") { + "integer" + } else if ["FLOAT", "DOUBLE", "REAL", "NUMERIC", "DECIMAL"] + .iter() + .any(|t| upper.contains(t)) + { + "floating" + } else if upper.contains("BOOL") { + "boolean" + } else if upper.contains("TIMESTAMP") || upper.contains("DATETIME") { + "datetime" + } else if upper.contains("DATE") { + "date" + } else if upper.contains("TIME") { + "time" + } else if ["CHAR", "TEXT", "STRING", "UUID"] + .iter() + .any(|t| upper.contains(t)) + { + "string" + } else { + "unknown" + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::results::filter::{Capabilities, Filter, SortKey}; + use crate::results::view::Rows; + use serde_json::json; + + const CAPS: Capabilities = Capabilities { regex: true }; + + fn frame(sql: &str) -> DataFrame { + ggsql::reader::connection::reader_from_uri("duckdb://memory") + .unwrap() + .execute_sql(sql) + .unwrap() + } + + fn numbers(n: usize) -> DataFrame { + frame(&format!( + "SELECT i AS n, i % 3 AS g, 'item ' || i AS s FROM range({n}) t(i)" + )) + } + + fn page( + store: &mut ResultStore, + view: &str, + first: usize, + last: usize, + col: usize, + ) -> Vec { + let (v, cache) = store.view_mut(view).unwrap(); + v.page(cache.backend(), &Rows::Range { first, last }, &[col]) + .unwrap() + .remove(0) + .into_iter() + .map(|v| v.as_str().unwrap_or("NULL").to_string()) + .collect() + } + + #[test] + fn test_open_describes_the_result_without_the_row_column() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(numbers(5), "Query [1]".into(), "SELECT 1".into()); + let opened = store.open(&key).unwrap(); + assert_eq!(opened.num_rows, 5); + let names: Vec<_> = opened.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["n", "g", "s"]); + assert_eq!(opened.columns[0].kind, "integer"); + assert_eq!(opened.columns[2].kind, "string"); + assert_eq!(page(&mut store, &opened.view, 1, 3, 0), ["1", "2", "3"]); + } + + #[test] + fn test_sort_is_stable_and_clearing_it_restores_the_order() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(numbers(9), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&view).unwrap(); + v.set_sort( + vec![SortKey { + column: 1, + ascending: false, + }], + cache.backend(), + ) + .unwrap(); + // g = 2 first, ties in the query's own order. + assert_eq!(page(&mut store, &view, 0, 2, 0), ["2", "5", "8"]); + assert_eq!(page(&mut store, &view, 3, 5, 0), ["1", "4", "7"]); + + let (v, cache) = store.view_mut(&view).unwrap(); + v.set_sort(vec![], cache.backend()).unwrap(); + assert_eq!(page(&mut store, &view, 0, 2, 0), ["0", "1", "2"]); + } + + #[test] + fn test_filters_select_rows_and_report_bad_ones() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(numbers(10), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + let filters: Vec = serde_json::from_value(json!([ + {"type": "compare", "column": 0, "op": ">=", "value": "4"}, + {"type": "compare", "column": 0, "op": "=", "value": "four"}, + {"type": "text", "column": 2, "match": "ends_with", "term": "7"} + ])) + .unwrap(); + let (v, cache) = store.view_mut(&view).unwrap(); + let errors = v.set_filters(filters, cache.backend(), CAPS).unwrap(); + assert!(errors[0].is_none()); + assert!(errors[1].as_deref().unwrap().contains("not a number")); + assert!(errors[2].is_none()); + assert_eq!(v.selected_rows, 1); + assert_eq!(page(&mut store, &view, 0, 0, 2), ["item 7"]); + } + + #[test] + fn test_dates_page_and_profile_as_dates() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert( + frame("SELECT DATE '2024-01-01' + i::INTEGER AS d FROM range(3) t(i)"), + "q".into(), + String::new(), + ); + let view = store.open(&key).unwrap().view; + assert_eq!( + page(&mut store, &view, 0, 1, 0), + ["2024-01-01", "2024-01-02"] + ); + let (v, cache) = store.view_mut(&view).unwrap(); + let req: view::ProfileRequest = serde_json::from_value(json!({"summary": true})).unwrap(); + let p = v.profile(cache.backend(), 0, &req).unwrap(); + assert_eq!(p["summary"]["min"], "2024-01-01"); + assert_eq!(p["summary"]["max"], "2024-01-03"); + } + + fn summary_of(sql: &str, column: usize) -> serde_json::Value { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(frame(sql), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&view).unwrap(); + let req: view::ProfileRequest = + serde_json::from_value(json!({"null_count": true, "summary": true})).unwrap(); + v.profile(cache.backend(), column, &req).unwrap() + } + + #[test] + fn test_stdev_keeps_its_precision_around_large_values() { + let p = summary_of("SELECT 1000000000 + i AS v FROM range(3) t(i)", 0); + let sd: f64 = p["summary"]["stdev"].as_str().unwrap().parse().unwrap(); + assert!((sd - 1.0).abs() < 1e-9, "{sd}"); + } + + #[test] + fn test_stdev_is_left_out_with_a_nan_or_infinity() { + for special in ["'nan'::DOUBLE", "'inf'::DOUBLE"] { + let p = summary_of( + &format!("SELECT * FROM (VALUES (1.0), (2.0), ({special})) t(v)"), + 0, + ); + assert!(p["summary"].get("stdev").is_none(), "{special}: {p}"); + } + } + + #[test] + fn test_nested_columns_are_not_numbers_and_still_profile() { + assert_eq!(kind_of("INTEGER[]"), "unknown"); + assert_eq!(kind_of("STRUCT(a INTEGER)"), "unknown"); + assert_eq!(kind_of("MAP(VARCHAR, INTEGER)"), "unknown"); + let p = summary_of("SELECT [1, 2] AS l, {'a': 1} AS s", 0); + assert_eq!(p["null_count"], 0); + assert_eq!(p["summary"], json!({})); + } + + #[test] + fn test_repeated_column_names_are_made_unique() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert( + frame("SELECT 1 AS a, 2 AS a, 3 AS a_1"), + "q".into(), + String::new(), + ); + let opened = store.open(&key).unwrap(); + let names: Vec<_> = opened.columns.iter().map(|c| c.name.as_str()).collect(); + assert_eq!(names, ["a", "a_2", "a_1"]); + assert_eq!(page(&mut store, &opened.view, 0, 0, 1), ["2"]); + } + + #[test] + fn test_a_failed_open_keeps_the_result() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(frame("SELECT 1 AS x"), "q".into(), String::new()); + // Break the cache's bookkeeping, so copying the result in fails. + store.cache.ensure_meta_table().unwrap(); + store + .cache + .backend() + .execute_sql(&format!( + "DROP TABLE {}", + quote_ident(ggsql::naming::CACHE_META_TABLE) + )) + .unwrap(); + for _ in 0..2 { + let err = store.open(&key).unwrap_err().to_string(); + // The real reason each time, not that the result is gone. + assert!(!err.contains("no longer available"), "{err}"); + } + } + + #[test] + fn test_sql_leaves_out_a_filter_the_cache_rejected() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert( + frame("SELECT 'a' AS s FROM range(3)"), + "q".into(), + "SELECT 1".into(), + ); + let view = store.open(&key).unwrap().view; + // A regex the cache refuses to compile. + let filters: Vec = serde_json::from_value(json!([ + {"type": "text", "column": 0, "match": "regex", "term": "(", "case_sensitive": true}, + {"type": "is_null", "column": 0, "negate": true} + ])) + .unwrap(); + let (v, cache) = store.view_mut(&view).unwrap(); + let errors = v.set_filters(filters, cache.backend(), CAPS).unwrap(); + assert!(errors[0].is_some() && errors[1].is_none(), "{errors:?}"); + let sql = v.to_sql("SELECT 1", &ggsql::reader::AnsiDialect, CAPS); + assert!(sql[0].starts_with("-- Filter left out"), "{sql:?}"); + assert!(!sql.iter().any(|l| l.contains("regexp_matches")), "{sql:?}"); + assert!(sql.iter().any(|l| l.contains("IS NOT NULL")), "{sql:?}"); + } + + #[test] + fn test_paging_past_the_end_returns_what_there_is() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(numbers(3), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + assert_eq!(page(&mut store, &view, 2, 10, 0), ["2"]); + } + + #[test] + fn test_profiles() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert( + frame("SELECT * FROM (VALUES (1, 'a'), (2, 'b'), (3, 'a'), (NULL, NULL)) t(n, s)"), + "q".into(), + String::new(), + ); + let view = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&view).unwrap(); + let req: view::ProfileRequest = serde_json::from_value(json!({ + "null_count": true, "summary": true, + "histogram": {"bins": 10, "quantiles": [0.5]} + })) + .unwrap(); + let p = v.profile(cache.backend(), 0, &req).unwrap(); + assert_eq!(p["null_count"], 1); + assert_eq!(p["summary"]["min"], "1"); + assert_eq!(p["summary"]["max"], "3"); + assert_eq!(p["summary"]["median"], "2"); + let counts: i64 = p["histogram"]["bin_counts"] + .as_array() + .unwrap() + .iter() + .map(|c| c.as_i64().unwrap()) + .sum(); + assert_eq!(counts, 3); + + let req: view::ProfileRequest = + serde_json::from_value(json!({"frequency": {"limit": 1}})).unwrap(); + let p = v.profile(cache.backend(), 1, &req).unwrap(); + assert_eq!(p["frequency"]["values"], json!(["a"])); + assert_eq!(p["frequency"]["counts"], json!([2])); + assert_eq!(p["frequency"]["other_count"], 1); + } + + #[test] + fn test_export_follows_the_view() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(numbers(4), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&view).unwrap(); + v.set_sort( + vec![SortKey { + column: 0, + ascending: false, + }], + cache.backend(), + ) + .unwrap(); + let csv = v + .export( + cache.backend(), + Some(&Rows::Range { first: 0, last: 1 }), + &[0, 2], + "csv", + ) + .unwrap(); + assert_eq!(csv, "n,s\n3,item 3\n2,item 2"); + } + + #[test] + fn test_an_open_result_survives_the_budget_and_a_closed_one_does_not() { + let mut store = ResultStore::with_max_bytes(4, 1).unwrap(); + let a = store.insert(numbers(3), "a".into(), String::new()); + let b = store.insert(numbers(3), "b".into(), String::new()); + let va = store.open(&a).unwrap().view; + let vb = store.open(&b).unwrap().view; + // Both open: both pinned, whatever the budget. + assert_eq!(page(&mut store, &va, 0, 0, 0), ["0"]); + assert_eq!(page(&mut store, &vb, 0, 0, 0), ["0"]); + + store.close(&va).unwrap(); + let err = store.open(&a).unwrap_err().to_string(); + assert!(err.contains("no longer available"), "{err}"); + assert!(store.open(&b).is_ok()); + } + + #[test] + fn test_running_out_of_memory_is_an_error_not_a_spill() { + let mut store = ResultStore::new(4).unwrap(); + let setting = store + .cache + .backend() + .execute_sql("SELECT current_setting('temp_directory') AS d") + .unwrap(); + assert_eq!( + ggsql::array_util::as_str(setting.column("d").unwrap()) + .unwrap() + .value(0), + "" + ); + + let key = store.insert( + frame("SELECT i AS n, md5(i::VARCHAR) AS s FROM range(200000) t(i)"), + "q".into(), + String::new(), + ); + let view = store.open(&key).unwrap().view; + // One thread, so the test leaves the machine to the others. + store + .cache + .backend() + .execute_sql("SET threads = 1") + .unwrap(); + // Just above what the copy uses, so the sort is what runs out. + let used = store + .cache + .backend() + .execute_sql("SELECT CAST(SUM(memory_usage_bytes) AS BIGINT) AS n FROM duckdb_memory()") + .unwrap(); + let used = ggsql::array_util::as_i64(used.column("n").unwrap()) + .unwrap() + .value(0); + store + .cache + .backend() + .execute_sql(&format!("SET memory_limit = '{}B'", used + 4_000_000)) + .unwrap(); + let (v, cache) = store.view_mut(&view).unwrap(); + let err = v + .set_sort( + vec![SortKey { + column: 1, + ascending: true, + }], + cache.backend(), + ) + .unwrap_err(); + assert!(err.to_lowercase().contains("memory"), "{err}"); + } + + #[test] + fn test_reopening_reuses_the_cached_copy() { + let mut store = ResultStore::new(4).unwrap(); + let key = store.insert(numbers(3), "q".into(), String::new()); + let first = store.open(&key).unwrap().view; + store.close(&first).unwrap(); + let second = store.open(&key).unwrap(); + assert_ne!(second.view, first); + assert_eq!(second.num_rows, 3); + } + + #[test] + fn test_the_oldest_unopened_result_is_dropped_past_the_limit() { + let mut store = ResultStore::new(2).unwrap(); + let first = store.insert(numbers(1), "a".into(), String::new()); + let second = store.insert(numbers(1), "b".into(), String::new()); + store.insert(numbers(1), "c".into(), String::new()); + assert!(store.open(&first).is_err()); + assert!(store.open(&second).is_ok()); + } + + #[test] + fn test_an_unknown_key_says_the_result_is_gone() { + let mut store = ResultStore::new(2).unwrap(); + let err = store.open("nope").err().unwrap().to_string(); + assert!(err.contains("no longer available"), "{err}"); + } + + #[test] + fn test_kind_of() { + assert_eq!(kind_of("BIGINT"), "integer"); + assert_eq!(kind_of("DECIMAL(18,3)"), "floating"); + assert_eq!(kind_of("TIMESTAMP WITH TIME ZONE"), "datetime"); + assert_eq!(kind_of("DATE"), "date"); + assert_eq!(kind_of("VARCHAR"), "string"); + assert_eq!(kind_of("INTERVAL"), "unknown"); + } +} diff --git a/ggsql-jupyter/src/results/view.rs b/ggsql-jupyter/src/results/view.rs new file mode 100644 index 000000000..0c4019681 --- /dev/null +++ b/ggsql-jupyter/src/results/view.rs @@ -0,0 +1,686 @@ +//! One view of a result: its filters and sort, and the rows they select. +//! +//! The result is a table in the results cache, with a [`ROW`] column holding +//! each row's position in the query's own output. Once a view is filtered or +//! sorted, it keeps an **order table** of `(pos, ROW)` pairs: `pos` is a row's +//! position in the view. A page then joins that small table back to the +//! result, so scrolling never sorts again. An unfiltered, unsorted view needs +//! no order table: its positions are the `ROW` values themselves. + +use super::filter::{self, Capabilities, Filter, SortKey}; +use super::{export, Column, ROW}; +use ggsql::naming::quote_ident; +use ggsql::reader::Reader; +use ggsql::sql::FromItem; +use ggsql::DataFrame; +use serde::Deserialize; +use serde_json::{json, Value}; + +/// The rows of a view to read. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum Rows { + /// `first..=last`. + Range { first: usize, last: usize }, + /// These positions, in this order. + Indices { indices: Vec }, +} + +impl Rows { + fn positions(&self) -> Vec { + match self { + Rows::Range { first, last } => (*first..=*last).collect(), + Rows::Indices { indices } => indices.clone(), + } + } +} + +/// Which profiles to compute for a column. +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ProfileRequest { + #[serde(default)] + pub null_count: bool, + #[serde(default)] + pub summary: bool, + pub histogram: Option, + pub frequency: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct HistogramRequest { + #[serde(default = "default_bins")] + pub bins: usize, + #[serde(default)] + pub quantiles: Vec, +} + +fn default_bins() -> usize { + 20 +} + +#[derive(Debug, Clone, Deserialize)] +pub struct FrequencyRequest { + #[serde(default = "default_limit")] + pub limit: usize, +} + +fn default_limit() -> usize { + 8 +} + +/// A view of a result table. +pub struct ResultView { + /// The result this views. + pub key: String, + /// The result table, quoted. + table: String, + /// The order table's name, used once the view is filtered or sorted. + order_name: String, + /// Whether the order table exists. + ordered: bool, + pub columns: Vec, + /// Rows in the result. + pub num_rows: usize, + /// Rows passing the filters. + pub selected_rows: usize, + /// The filters as last set, valid or not. + pub filters: Vec, + /// Why each of `filters` was left out, or `None` for one that applies. + rejected: Vec>, + /// The valid filters ANDed together, or `None` when unfiltered. + predicate: Option, + pub sort: Vec, +} + +impl ResultView { + pub fn new(key: &str, id: &str, table: &str, columns: Vec, num_rows: usize) -> Self { + Self { + key: key.to_string(), + table: quote_ident(table), + order_name: format!("__ggsql_view_{id}__"), + ordered: false, + columns, + num_rows, + selected_rows: num_rows, + filters: Vec::new(), + rejected: Vec::new(), + predicate: None, + sort: Vec::new(), + } + } + + /// The result table filtered, as a `FROM` item, for the queries that + /// aggregate it. + fn relation(&self) -> String { + match &self.predicate { + None => self.table.clone(), + Some(p) => format!( + "(SELECT * FROM {} WHERE {}) AS \"__ggsql_filtered__\"", + self.table, p + ), + } + } + + /// Set the row filters, returning an error message per filter (`None` for + /// a valid one). A filter that cannot be built, or that the database + /// rejects, is left out rather than failing the rest. + pub fn set_filters( + &mut self, + filters: Vec, + reader: &dyn Reader, + caps: Capabilities, + ) -> Result>, String> { + let mut errors: Vec> = vec![None; filters.len()]; + let mut built = Vec::new(); + for (i, f) in filters.iter().enumerate() { + match filter::predicate(f, &self.columns, reader.dialect(), caps) { + Ok(p) => built.push((i, p)), + Err(e) => errors[i] = Some(e), + } + } + + let combine = |built: &[(usize, String)]| { + (!built.is_empty()).then(|| { + built + .iter() + .map(|(_, p)| format!("({p})")) + .collect::>() + .join(" AND ") + }) + }; + if count_rows(reader, &self.table, combine(&built).as_deref()).is_err() { + // Find the filters the database rejects. + built.retain(|(i, p)| match count_rows(reader, &self.table, Some(p)) { + Ok(_) => true, + Err(e) => { + errors[*i] = Some(e); + false + } + }); + } + + self.filters = filters; + self.rejected = errors.clone(); + self.predicate = combine(&built); + self.rebuild(reader)?; + Ok(errors) + } + + /// Set the sort keys. + pub fn set_sort(&mut self, keys: Vec, reader: &dyn Reader) -> Result<(), String> { + filter::order_terms(&keys, &self.columns)?; + self.sort = keys; + self.rebuild(reader) + } + + /// Rebuild the order table for the current filters and sort, and count + /// the rows they select. + fn rebuild(&mut self, reader: &dyn Reader) -> Result<(), String> { + self.drop_order(reader); + if self.predicate.is_none() && self.sort.is_empty() { + self.selected_rows = self.num_rows; + return Ok(()); + } + + // Ties break on the query's own order, so paging is stable and clearing + // the sort restores it. + let mut terms = filter::order_terms(&self.sort, &self.columns)?; + terms.push(quote_ident(ROW)); + let filter = self + .predicate + .as_ref() + .map(|p| format!(" WHERE {p}")) + .unwrap_or_default(); + let body = format!( + "SELECT ROW_NUMBER() OVER (ORDER BY {}) - 1 AS \"pos\", {row} FROM {}{filter}", + terms.join(", "), + self.table, + row = quote_ident(ROW), + ); + let dialect = reader.dialect(); + for stmt in dialect.create_or_replace_temp_table_sql(&self.order_name, &[], &body) { + reader.execute_sql(&stmt).map_err(|e| e.to_string())?; + } + self.ordered = true; + self.selected_rows = count_rows(reader, "e_ident(&self.order_name), None)?; + Ok(()) + } + + fn drop_order(&mut self, reader: &dyn Reader) { + if self.ordered { + let sql = reader.dialect().drop_table_sql(&self.order_name); + if let Err(e) = reader.execute_sql(&sql) { + tracing::warn!("Failed to drop {}: {}", self.order_name, e); + } + self.ordered = false; + } + } + + /// Drop what the view keeps in the cache. + pub fn close(&mut self, reader: &dyn Reader) { + self.drop_order(reader); + } + + /// Read the view's rows `first..=last`, or every row, for `cols`. + fn fetch( + &self, + reader: &dyn Reader, + cols: &[usize], + span: Option<(usize, usize)>, + ) -> Result { + let names = cols + .iter() + .map(|&i| { + self.columns + .get(i) + .map(|c| format!("t.{}", quote_ident(&c.name))) + .ok_or_else(|| format!("There is no column {i}")) + }) + .collect::, _>>()?; + if names.is_empty() { + return Ok(DataFrame::empty()); + } + let row = quote_ident(ROW); + let sql = if self.ordered { + let range = span + .map(|(a, b)| format!(" WHERE o.\"pos\" BETWEEN {a} AND {b}")) + .unwrap_or_default(); + format!( + "SELECT {} FROM {} AS o JOIN {} AS t ON t.{row} = o.{row}{range} ORDER BY o.\"pos\"", + names.join(", "), + quote_ident(&self.order_name), + self.table, + ) + } else { + let range = span + .map(|(a, b)| format!(" WHERE t.{row} BETWEEN {a} AND {b}")) + .unwrap_or_default(); + format!( + "SELECT {} FROM {} AS t{range} ORDER BY t.{row}", + names.join(", "), + self.table, + ) + }; + reader.execute_sql(&sql).map_err(|e| e.to_string()) + } + + /// The cells at `rows` of each of `cols`, as `columns[col][row]`: a string, + /// or `null` for a null. + pub fn page( + &self, + reader: &dyn Reader, + rows: &Rows, + cols: &[usize], + ) -> Result>, String> { + let positions = rows.positions(); + let (Some(&first), Some(&last)) = (positions.iter().min(), positions.iter().max()) else { + return Ok(vec![Vec::new(); cols.len()]); + }; + let df = self.fetch(reader, cols, Some((first, last)))?; + Ok(df + .get_columns() + .iter() + .map(|col| { + positions + .iter() + .map(|&r| r - first) + .filter(|&r| r < df.height()) + .map(|r| cell_value(col, r)) + .collect() + }) + .collect()) + } + + /// The cells at `rows` (every row when `None`) of `cols` as CSV, TSV or + /// HTML, with a header row of column names. + pub fn export( + &self, + reader: &dyn Reader, + rows: Option<&Rows>, + cols: &[usize], + format: &str, + ) -> Result { + let (df, picked): (DataFrame, Vec) = match rows { + None => { + let df = self.fetch(reader, cols, None)?; + let n = df.height(); + (df, (0..n).collect()) + } + Some(rows) => { + let positions = rows.positions(); + let first = positions.iter().min().copied().unwrap_or(0); + let last = positions.iter().max().copied().unwrap_or(0); + let df = self.fetch(reader, cols, Some((first, last)))?; + let n = df.height(); + let picked = positions + .iter() + .map(|r| r - first) + .filter(|&r| r < n) + .collect(); + (df, picked) + } + }; + let cells: Vec> = picked + .iter() + .map(|&r| df.get_columns().iter().map(|c| cell_string(c, r)).collect()) + .collect(); + let header: Vec = cols.iter().map(|&i| self.columns[i].name.clone()).collect(); + export::format(Some(&header), &cells, format) + } + + /// SQL for `dialect` reproducing this view over the result's `query`. + pub fn to_sql( + &self, + query: &str, + dialect: &dyn ggsql::reader::SqlDialect, + caps: Capabilities, + ) -> Vec { + // Only the filters the view applies: one the cache rejected would make + // the SQL fail, or select rows the view does not show. + let mut applied = Vec::new(); + let mut lines = Vec::new(); + for (i, f) in self.filters.iter().enumerate() { + match self.rejected.get(i).cloned().flatten() { + None => applied.push(f.clone()), + Some(e) => lines.push(format!("-- Filter left out: {e}")), + } + } + lines.extend(filter::to_sql( + &applied, + &self.sort, + &filter::subquery(query), + &self.columns, + dialect, + caps, + )); + lines + } + + /// Profiles of one column over the view's filtered rows. + pub fn profile( + &self, + reader: &dyn Reader, + index: usize, + request: &ProfileRequest, + ) -> Result { + let col = self + .columns + .get(index) + .ok_or_else(|| format!("There is no column {index}"))?; + let mut out = json!({}); + let kind = col.kind.as_str(); + + if request.null_count { + let sql = format!( + "SELECT SUM(CASE WHEN {} IS NULL THEN 1 ELSE 0 END) AS \"n\" FROM {}", + quote_ident(&col.name), + self.relation() + ); + out["null_count"] = json!(scalar_i64(reader, &sql).unwrap_or(0)); + } + if request.summary { + // A column the summary cannot handle still gets the rest of its + // profile, and an empty summary rather than none, which a + // frontend would wait on. + out["summary"] = self.summary(reader, col).unwrap_or_else(|e| { + tracing::warn!("Summarising column {} failed: {}", col.name, e); + json!({}) + }); + } + if let Some(h) = &request.histogram { + if matches!(kind, "integer" | "floating") { + if let Some(hist) = self.histogram(reader, col, h) { + out["histogram"] = hist; + } + } + } + if let Some(f) = &request.frequency { + if matches!(kind, "string" | "boolean") { + if let Some(freq) = self.frequency(reader, col, f.limit) { + out["frequency"] = freq; + } + } + } + Ok(out) + } + + /// Summary statistics, with fields that depend on the column's kind. + fn summary(&self, reader: &dyn Reader, col: &Column) -> Result { + let dialect = reader.dialect(); + let c = quote_ident(&col.name); + let float = dialect.type_names().number.unwrap_or("DOUBLE PRECISION"); + let parts: Vec = match col.kind.as_str() { + "integer" | "floating" => vec![ + format!("MIN({c}) AS \"min\""), + format!("MAX({c}) AS \"max\""), + format!("AVG(CAST({c} AS {float})) AS \"mean\""), + ], + "boolean" => vec![ + format!( + "SUM(CASE WHEN {c} = {} THEN 1 ELSE 0 END) AS \"true_count\"", + dialect.sql_boolean_literal(true) + ), + format!( + "SUM(CASE WHEN {c} = {} THEN 1 ELSE 0 END) AS \"false_count\"", + dialect.sql_boolean_literal(false) + ), + ], + "string" => vec![ + format!("COUNT(DISTINCT {c}) AS \"num_unique\""), + format!("SUM(CASE WHEN {c} = '' THEN 1 ELSE 0 END) AS \"num_empty\""), + ], + "date" | "datetime" => vec![ + format!("MIN({c}) AS \"min\""), + format!("MAX({c}) AS \"max\""), + format!("COUNT(DISTINCT {c}) AS \"num_unique\""), + ], + _ => return Ok(json!({})), + }; + let sql = format!("SELECT {} FROM {}", parts.join(", "), self.relation()); + let df = reader.execute_sql(&sql).map_err(|e| e.to_string())?; + let get = |name: &str| first_string(&df, name); + let get_i64 = |name: &str| get(name).and_then(|s| s.parse::().ok()); + + let mut out = json!({}); + match col.kind.as_str() { + "integer" | "floating" => { + for name in ["min", "max", "mean"] { + if let Some(v) = get(name) { + out[name] = json!(v); + } + } + if let Some(sd) = self.stdev(reader, col) { + out["stdev"] = json!(sd); + } + if let Some(m) = self.quantile(reader, col, 0.5) { + out["median"] = json!(m); + } + } + "boolean" => { + out["true_count"] = json!(get_i64("true_count").unwrap_or(0)); + out["false_count"] = json!(get_i64("false_count").unwrap_or(0)); + } + "string" => { + out["num_unique"] = json!(get_i64("num_unique").unwrap_or(0)); + out["num_empty"] = json!(get_i64("num_empty").unwrap_or(0)); + } + _ => { + for name in ["min", "max"] { + if let Some(v) = get(name) { + out[name] = json!(v); + } + } + if let Some(n) = get_i64("num_unique") { + out["num_unique"] = json!(n); + } + } + } + Ok(out) + } + + /// The sample standard deviation of a numeric column, computed by the + /// database rather than from sums, which lose every digit of a small spread + /// around large values. Left out when there is none to report: fewer than + /// two values, or a NaN or infinity among them. + fn stdev(&self, reader: &dyn Reader, col: &Column) -> Option { + let float = reader + .dialect() + .type_names() + .number + .unwrap_or("DOUBLE PRECISION"); + let sql = format!( + "SELECT STDDEV_SAMP(CAST({} AS {float})) AS \"sd\" FROM {}", + quote_ident(&col.name), + self.relation() + ); + let df = reader.execute_sql(&sql).ok()?; + let sd = first_string(&df, "sd")?; + sd.parse::().ok().filter(|v| v.is_finite())?; + Some(sd) + } + + /// The `q` quantile of a column, through the dialect's quantile hook. + fn quantile(&self, reader: &dyn Reader, col: &Column, q: f64) -> Option { + // The expression must sit in a query over `from`: native dialects + // return a bare aggregate, which without a FROM computes over nothing. + let from = format!("SELECT * FROM {}", self.relation()); + let expr = reader + .dialect() + .sql_quantile(&col.name, q, FromItem::Query(&from), &[]); + let sql = format!("SELECT {expr} AS \"q\" FROM ({from}) AS \"__ggsql_qt__\""); + let df = reader.execute_sql(&sql).ok()?; + first_string(&df, "q") + } + + /// A histogram of a numeric column: Sturges' rule for the bin count, + /// capped at the request and, for integers, at one bin per value. + fn histogram( + &self, + reader: &dyn Reader, + col: &Column, + req: &HistogramRequest, + ) -> Option { + if req.bins == 0 { + return None; + } + let float = reader + .dialect() + .type_names() + .number + .unwrap_or("DOUBLE PRECISION"); + let c = quote_ident(&col.name); + let bounds = reader + .execute_sql(&format!( + "SELECT MIN(CAST({c} AS {float})) AS \"min\", MAX(CAST({c} AS {float})) AS \"max\", \ + COUNT({c}) AS \"cnt\" FROM {} WHERE {c} IS NOT NULL", + self.relation() + )) + .ok()?; + let min: f64 = first_string(&bounds, "min")?.parse().ok()?; + let max: f64 = first_string(&bounds, "max")?.parse().ok()?; + let count: usize = first_string(&bounds, "cnt") + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let quantiles: Vec = req + .quantiles + .iter() + .filter_map(|&q| { + self.quantile(reader, col, q) + .map(|v| json!({"q": q, "value": v})) + }) + .collect(); + + if (max - min).abs() < f64::EPSILON { + return Some(json!({ + "bin_edges": [min.to_string(), max.to_string()], + "bin_counts": [count], + "quantiles": quantiles + })); + } + + let mut bins = if count > 1 { + (count as f64).log2().ceil() as usize + 1 + } else { + 1 + }; + if col.kind == "integer" { + bins = bins.min((max - min) as usize + 1); + } + let bins = bins.min(req.bins).max(1); + let width = (max - min) / bins as f64; + + // The maximum falls in the last bin rather than one past it. + let sql = format!( + "SELECT CASE WHEN \"bin\" >= {bins} THEN {last} ELSE \"bin\" END AS \"b\", COUNT(*) AS \"n\" \ + FROM (SELECT FLOOR((CAST({c} AS {float}) - {min}) / {width}) AS \"bin\" \ + FROM {} WHERE {c} IS NOT NULL) AS \"__bins__\" \ + GROUP BY \"b\" ORDER BY \"b\"", + self.relation(), + last = bins - 1, + ); + let df = reader.execute_sql(&sql).ok()?; + let mut counts = vec![0i64; bins]; + let (b, n) = (df.column("b").ok()?, df.column("n").ok()?); + for i in 0..df.height() { + let bin = ggsql::array_util::value_to_string(b, i) + .parse::() + .ok() + .map(|v| v as usize); + if let Some(bin) = bin.filter(|&bin| bin < bins) { + counts[bin] = ggsql::array_util::value_to_string(n, i) + .parse() + .unwrap_or(0); + } + } + let edges: Vec = (0..=bins) + .map(|i| (min + i as f64 * width).to_string()) + .collect(); + Some(json!({ "bin_edges": edges, "bin_counts": counts, "quantiles": quantiles })) + } + + /// The `limit` most frequent values of a column, and the count of the rest. + fn frequency(&self, reader: &dyn Reader, col: &Column, limit: usize) -> Option { + let c = quote_ident(&col.name); + let df = reader + .execute_sql(&format!( + "SELECT {c} AS \"value\", COUNT(*) AS \"count\" FROM {} WHERE {c} IS NOT NULL \ + GROUP BY {c} ORDER BY COUNT(*) DESC, {c} LIMIT {limit}", + self.relation() + )) + .ok()?; + let (v, n) = (df.column("value").ok()?, df.column("count").ok()?); + let values: Vec = (0..df.height()).map(|i| cell_string(v, i)).collect(); + let counts: Vec = (0..df.height()) + .map(|i| { + ggsql::array_util::value_to_string(n, i) + .parse() + .unwrap_or(0) + }) + .collect(); + let total = scalar_i64( + reader, + &format!("SELECT COUNT({c}) AS \"n\" FROM {}", self.relation()), + ) + .unwrap_or(0); + let other = total - counts.iter().sum::(); + Some(json!({ "values": values, "counts": counts, "other_count": other })) + } +} + +/// `SELECT COUNT(*)` over `table`, filtered by `predicate` when given. +pub fn count_rows( + reader: &dyn Reader, + table: &str, + predicate: Option<&str>, +) -> Result { + let filter = predicate.map(|p| format!(" WHERE {p}")).unwrap_or_default(); + let n = scalar_i64( + reader, + &format!("SELECT COUNT(*) AS \"n\" FROM {table}{filter}"), + )?; + Ok(n.max(0) as usize) +} + +/// The first column of a one-row query as an integer. +fn scalar_i64(reader: &dyn Reader, sql: &str) -> Result { + let df = reader.execute_sql(sql).map_err(|e| e.to_string())?; + let name = df + .get_column_names() + .into_iter() + .next() + .ok_or("The query returned no columns")?; + Ok(first_string(&df, &name) + .and_then(|s| s.parse().ok()) + .unwrap_or(0)) +} + +/// Row 0 of `name` as text, or `None` when absent or null. +fn first_string(df: &DataFrame, name: &str) -> Option { + use arrow::array::Array; + let c = df.column(name).ok()?; + if c.is_empty() || c.is_null(0) { + None + } else { + Some(cell_string(c, 0)) + } +} + +/// A cell as text, with a null as empty. +fn cell_string(col: &arrow::array::ArrayRef, row: usize) -> String { + use arrow::array::Array; + if col.is_null(row) { + return String::new(); + } + crate::display::cell_text(col, row) +} + +/// A cell for a page: its text, or `null`. +fn cell_value(col: &arrow::array::ArrayRef, row: usize) -> Value { + use arrow::array::Array; + if col.is_null(row) { + Value::Null + } else { + Value::String(cell_string(col, row)) + } +} diff --git a/ggsql-jupyter/tests/test_integration.py b/ggsql-jupyter/tests/test_integration.py index a409ec156..83fee6e66 100644 --- a/ggsql-jupyter/tests/test_integration.py +++ b/ggsql-jupyter/tests/test_integration.py @@ -17,7 +17,7 @@ # the kernel is built once rather than once per test file. -def _launch(kernel_binary, extra_args=()): +def _launch(kernel_binary, extra_args=(), env=None): """Start a kernel and yield its manager. Shared by the fixtures below.""" kernel_process = None km = None @@ -37,6 +37,7 @@ def _launch(kernel_binary, extra_args=()): # Start our kernel process directly kernel_process = subprocess.Popen( [kernel_binary, "-f", connection_file, *extra_args], + env={**os.environ, **(env or {})}, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) @@ -507,6 +508,156 @@ def test_an_unknown_method_is_an_error_not_a_null_result(self, console_client): assert "result" not in data +class TestResultsApi: + """Console results kept for the results API, called through `call_method` + on the UI comm as `session.callMethod` sends it.""" + + QUERY = "SELECT i AS n, 'r' || i AS label FROM range(5) t(i)" + UI = "test-ui-comm" + + def _result_key(self, client): + """Run QUERY and return the result key in its execute_result.""" + client.execute(self.QUERY, silent=False, store_history=True) + messages = drain_iopub(client) + reply = client.get_shell_msg(timeout=5) + assert reply["content"]["status"] == "ok" + results = [m for m in messages if m["msg_type"] == "execute_result"] + assert len(results) == 1, [m["msg_type"] for m in messages] + # The kernel prints no link of its own: the extension adds it. + assert not [m for m in messages if m["msg_type"] == "stream"] + return results[0]["content"]["metadata"]["ggsql"]["result"] + + def _open_ui(self, client): + client.session.send( + client.shell_channel.socket, + "comm_open", + {"comm_id": self.UI, "target_name": "positron.ui", "data": {}}, + ) + drain_iopub(client) + + def _close_ui(self, client): + client.session.send( + client.shell_channel.socket, "comm_close", {"comm_id": self.UI, "data": {}} + ) + drain_iopub(client) + + def _call(self, client, method, params): + """`callMethod(method, params)`: the reply's JSON-RPC data.""" + client.session.send( + client.shell_channel.socket, + "comm_msg", + { + "comm_id": self.UI, + "data": { + "jsonrpc": "2.0", + "id": f"{method}-1", + "method": "call_method", + "params": {"method": method, "params": [params]}, + }, + }, + ) + reply = client.get_shell_msg(timeout=5) + drain_iopub(client) + return reply["content"]["data"] + + def test_open_filter_sort_page_and_close(self, console_client): + key = self._result_key(console_client) + self._open_ui(console_client) + try: + opened = self._call(console_client, "result_open", {"key": key})["result"] + assert opened["title"].startswith("Query [") + assert opened["num_rows"] == 5 + assert [c["name"] for c in opened["columns"]] == ["n", "label"] + view = opened["view"] + + filtered = self._call( + console_client, + "result_set_filters", + { + "view": view, + "filters": [{"type": "compare", "column": 0, "op": ">=", "value": "2"}], + }, + )["result"] + assert filtered == {"num_rows": 3, "errors": [None]} + + self._call( + console_client, + "result_set_sort", + {"view": view, "keys": [{"column": 0, "ascending": False}]}, + ) + page = self._call( + console_client, + "result_page", + {"view": view, "rows": {"first": 0, "last": 2}, "columns": [1]}, + )["result"] + assert page == {"columns": [["r4", "r3", "r2"]]} + + sql = self._call(console_client, "result_sql", {"view": view})["result"]["sql"] + assert f" {self.QUERY}" in sql + assert 'ORDER BY "n" DESC' in sql + + assert self._call(console_client, "result_close", {"view": view})["result"] == {} + # The cached copy is still there to reopen. + again = self._call(console_client, "result_open", {"key": key})["result"] + assert again["num_rows"] == 5 + finally: + self._close_ui(console_client) + + def test_an_unknown_key_is_a_json_rpc_error(self, console_client): + self._open_ui(console_client) + try: + data = self._call(console_client, "result_open", {"key": "deadbeef"}) + assert "error" in data and "result" not in data, data + assert "no longer available" in data["error"]["message"] + finally: + self._close_ui(console_client) + + def test_an_unknown_call_method_is_an_error(self, console_client): + self._open_ui(console_client) + try: + data = self._call(console_client, "not_a_results_method", {}) + assert "error" in data, data + finally: + self._close_ui(console_client) + + def test_only_a_console_keeps_results(self, client): + client.execute(self.QUERY, silent=False, store_history=True) + messages = drain_iopub(client) + client.get_shell_msg(timeout=5) + result = next(m for m in messages if m["msg_type"] == "execute_result") + assert "ggsql" not in result["content"]["metadata"] + + +class TestConsoleRows: + """`GGSQL_CONSOLE_ROWS`, which the extension sets from `ggsql.consoleRows`.""" + + def _table(self, client, n): + client.execute(f"SELECT i FROM range({n}) t(i)") + messages = drain_iopub(client) + client.get_shell_msg(timeout=5) + result = next(m for m in messages if m["msg_type"] == "execute_result") + return result["content"]["data"]["text/html"] + + def test_the_environment_sets_the_rows_a_console_shows(self, kernel_binary): + with _launch_kernel( + kernel_binary, + ["--session-mode", "console"], + env={"GGSQL_CONSOLE_ROWS": "3"}, + ) as km: + kc = km.client() + kc.start_channels() + try: + kc.wait_for_ready(timeout=10) + # Up to twice the setting, the whole result. + assert self._table(kc, 6).count("") == 1 + 6 + # Past it, the first 3 and a count of the rest. + html = self._table(kc, 7) + assert html.count("") == 1 + 3 + 1, html + assert "... 4 more rows" in html + finally: + kc.stop_channels() + + class TestStatus: """Test status messages.""" diff --git a/ggsql-vscode/AGENTS.md b/ggsql-vscode/AGENTS.md index 5cd314e17..e18bbb40f 100644 --- a/ggsql-vscode/AGENTS.md +++ b/ggsql-vscode/AGENTS.md @@ -19,6 +19,8 @@ ggsql-vscode/ │ ├── manager.ts Kernel discovery + Positron language-runtime registration │ ├── positronApi.ts Acquires the Positron API so Positron can attribute it to this extension │ ├── connections.ts Connection-string handling for the Connections pane +│ ├── resultLinks.ts Session wrapper adding "Open in Data Explorer" links to console results +│ ├── dataExplorer/ Data Explorer backend for console results (Positron protocol → kernel results API) │ ├── cellParser.ts Splits .ggsql files into cells for Run-Cell commands │ ├── codelens.ts "▶ Run cell" lens above each cell │ ├── decorations.ts Cell separator decorations @@ -99,7 +101,11 @@ The extension declares `contributes.languageRuntimes` for `ggsql` (see `package. - **`createKernelSpec` appends `--session-mode `** from `sessionMetadata.sessionMode`. The kernel keys every output decision off that: a console session gets the comm, a notebook session gets a static image in its cell, and background gets the standalone path. Without the flag the kernel falls back to guessing from its session id, which works but is a heuristic. `restoreSession` deliberately does *not* rebuild the spec — the supervisor replays the argv the session was created with, and a session's mode never changes. - **`uiSubscriptions: ['did_change_plots_render_settings']`** on the runtime metadata. Without it the frontend never tells the kernel how large the Plots pane is, and the kernel needs that for exactly one thing: rendering a *new* plot at the right size so its `comm_open` can carry it, and the pane shows it immediately instead of blank. Every other render carries its own size on the request. -Both live in `manager.ts`. The kernel side of the contract — why the comm alone creates the pane entry, and why an output bundle alongside it would show the plot twice — is in [`/ggsql-jupyter/AGENTS.md`](../ggsql-jupyter/AGENTS.md). +Both live in `manager.ts`, as does one piece of user configuration: **`ggsql.consoleRows` reaches the kernel as `GGSQL_CONSOLE_ROWS` in the spec's `env`, not as `--console-rows`.** A kernel found on the machine may predate the option, and it ignores an unknown variable where it would refuse to start on an unknown flag. `consoleRowsEnv` leaves out a value the kernel would reject, for the same reason. Like the session mode, it is fixed when a session starts. + +The kernel side of the contract — why the comm alone creates the pane entry, and why an output bundle alongside it would show the plot twice — is in [`/ggsql-jupyter/AGENTS.md`](../ggsql-jupyter/AGENTS.md). + +**Sessions are wrapped to link console results to the Data Explorer** — see [Console results in the Data Explorer](#console-results-in-the-data-explorer) below. Outside Positron there is no way to execute a query: `activate()` returns early, so every command that runs code stays unregistered. To avoid offering actions that cannot work, everything execution-related gates on Positron's built-in **`isPositron`** context key ([extension development docs](https://positron.posit.co/extension-development.html#option-1-context-keys)): @@ -117,6 +123,26 @@ The Positron Supervisor is a soft dependency, reached through `getSupervisorApi( Anything that does *not* need the runtime (`ggsql.createNewFile`, `ggsql.resetSqlAssociationPrompt`, syntax highlighting) is registered before the early return and works in plain VS Code. Add new commands on the correct side of that line, and gate them if they execute code. +### Console results in the Data Explorer + +A console query result opens in Positron's Data Explorer, served by a backend that lives in this extension rather than in the kernel. The kernel exposes a frontend-neutral **results API** — `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`, reached through `session.callMethod` — and the extension translates Positron's Data Explorer protocol to it. The kernel side, including what each method takes and returns, is in [`/ggsql-jupyter/AGENTS.md`](../ggsql-jupyter/AGENTS.md) and `ggsql-jupyter/src/results/methods.rs`. + +**The backend** (`src/dataExplorer/`) registers with `positron.dataExplorer.registerRpcHandler('ggsql', …)` from `extension.ts`, before the runtime manager, which is handed it as `resultExplorer`. `package.json` carries the `onPositronDataExplorerBackend:ggsql` activation event, which Positron fires before routing a request to a provider that has not registered yet — for example a Data Explorer restored by a window reload. + +- A dataset id is `ggsql:/`. The scheme is required: Positron puts the id in the explorer editor's URI and parses it back out (`PositronDataExplorerUri.parse`), accepting only a UUID or `:`, and an id it cannot parse opens an empty editor. `ResultExplorer` keeps a registry of the sessions the manager wrapped, and each open dataset holds one kernel view: opened with `result_open` on the first request or when a link is clicked, closed with `result_close` from `closeDataset`. The views of a session are forgotten when its kernel exits or restarts, since they ended with it, and each open explorer is sent `schema_update`, so it asks for its state again: the result reopens, or the explorer shows the kernel's reason it is gone. A `get_state` for a result that cannot be opened answers a disconnected state carrying the kernel's message, which the Data Explorer shows in place of the table. +- `handler.ts` dispatches the requests. Row filters and sort keys are remembered as last set, for `get_state`; filters are ANDed by the kernel, so `supports_conditions` is unsupported, and a filter that cannot be translated or that the kernel rejects goes back with `is_valid: false` and its `error_message`. `get_data_values` makes one `result_page` per distinct row spec. `get_column_profiles` replies `{}` at once and answers through the registration's `sendUiEvent` as `return_column_profiles`. `convert_to_code` is the kernel's `result_sql`, written in the user's own dialect. Column filters and row labels are unsupported. +- `filters.ts` maps Positron's `RowFilter`s and sort keys to the kernel's and runs `search_schema` over the columns `result_open` returned. `profiles.ts` maps profile requests and results; the kernel computes one histogram and one frequency table per call, so a large variant gets a call of its own. `format.ts` turns a kernel cell into Positron's `ColumnValue`: a null and, in a floating column only, `NaN`/`inf`/`-inf` become Positron's special-value codes. +- `protocol.ts` is a hand-written subset of Positron's protocol types, not a copy of Positron's generated (Elastic-licensed) file, plus the kernel's API types. +- Kernel errors arrive as rejected plain objects with a `message`, not `Error`s; `errorMessage()` reads both, and every request's error becomes the response's `error_message`. + +**The link** is added by the extension, not the kernel. In a Positron console the kernel marks each tabular `execute_result` with `{"ggsql": {"result": ""}}` in its metadata, which the supervisor exposes as the Result message's `outputMetadata`. `createSession` and `restoreSession` return the supervisor's session through `withResultLinks` (`src/resultLinks.ts`), a `Proxy` that: + +- **re-fires `onDidReceiveRuntimeMessage`**, in order, and after a marked Result in a *console-mode* session fires a synthesized stdout stream message (same `parent_id` and `when`, an id derived from the result's) holding an OSC 8 hyperlink to `x-ggsql-result:/`. Notebook and background sessions get no link: a notebook has no console to click in. +- **adds `openResource`**, which the supervisor's session does not implement. A clicked link opens the kernel view first — for the title, and so an expired result is reported as a warning before a tab opens — then calls `positron.dataExplorer.open`. Positron offers a clicked console hyperlink only to the *last active* console of each language, so the link carries the session id and is routed by it rather than to the session that was asked. Only a hex key is accepted. +- binds everything else to the underlying session, and on `dispose` releases its own subscriptions and the registry entry. + +When `positron.dataExplorer` is missing (a Positron without extension-hosted Data Explorer backends) nothing is registered and sessions are returned unwrapped, so no link is shown that could not be served. + ## Finding the kernel The extension ships the kernel: the per-platform VSIXes carry `ggsql-jupyter` at `bundled/bin/`, so installing the extension is enough and no native installer is needed. The platform-neutral VSIX carries none, and users on a platform without a build install the kernel themselves. @@ -151,7 +177,8 @@ The one case that interrupts the user is the dead end: nothing runnable anywhere ```json { - "ggsql.kernelPath": "string" // an extra kernel to offer, listed first + "ggsql.kernelPath": "string", // an extra kernel to offer, listed first + "ggsql.consoleRows": 10 // rows of a long result a console shows } ``` @@ -195,7 +222,7 @@ Tests live in `src/test/` and compile to `out-test/` via `tsconfig.test.json`, d Note that `tsc` does not prune output for deleted sources: if you delete or rename a test, remove its `.js` and `.js.map` from `out-test/test/` or the runner keeps executing the stale copy. `npm run test:extension` on its own does not recompile, so run `npm test` (or `npm run compile-tests` first) after editing any `.ts`. -The suites cover the extension as stock VS Code sees it: activation, language resolution, cell parsing, `.sql` gating, CodeLens placement, TextMate scopes, kernel discovery, and the parts of `manager.ts` and `positronApi.ts` that are reachable without a Positron host. `bundle.test.ts` additionally asserts against the built `out/extension.js`. The rest of the Positron surface (session creation, connection drivers, cell execution) is not covered, since it needs a Positron host, and `sqlAssociation.ts` and `connections.ts` are untested. +The suites cover the extension as stock VS Code sees it: activation, language resolution, cell parsing, `.sql` gating, CodeLens placement, TextMate scopes, kernel discovery, and the parts of `manager.ts` and `positronApi.ts` that are reachable without a Positron host. `bundle.test.ts` additionally asserts against the built `out/extension.js`. `dataExplorer.test.ts` drives the Data Explorer backend against a recorded stand-in for `callMethod`, and `resultLinks.test.ts` runs the session wrapper over a stand-in session's messages; neither needs Positron. The rest of the Positron surface (session creation, connection drivers, cell execution) is not covered, since it needs a Positron host, and `sqlAssociation.ts` and `connections.ts` are untested. Add new tests as `src/test/.test.ts`; no config change is needed. `.vscode-test.mjs` globs `out-test/test/*.test.js` — one level only, deliberately, so the Positron suite in `test/integration/` does not run under stock VS Code, where it cannot pass. diff --git a/ggsql-vscode/package.json b/ggsql-vscode/package.json index 4384361fe..11114e4fe 100644 --- a/ggsql-vscode/package.json +++ b/ggsql-vscode/package.json @@ -9,7 +9,8 @@ }, "main": "./out/extension.js", "activationEvents": [ - "onStartupFinished" + "onStartupFinished", + "onPositronDataExplorerBackend:ggsql" ], "categories": [ "Programming Languages" @@ -164,6 +165,12 @@ "default": "", "markdownDescription": "Path to a `ggsql-jupyter` executable to offer alongside the kernels found automatically, listed ahead of them. A bare name is looked up on `PATH`. Leave this empty unless you have a kernel in a place the extension does not look: it ships one, and it finds the ones installed on this machine." }, + "ggsql.consoleRows": { + "type": "integer", + "default": 10, + "minimum": 1, + "markdownDescription": "How many rows of a query result the ggsql console shows. A result with up to twice this many rows is shown in full; a longer one shows this many, then a count of the rest. Every result has a link beneath it that opens the whole result in the Data Explorer. Notebooks are unaffected. Takes effect in consoles started after the change." + }, "ggsql.enableSqlFiles": { "type": "boolean", "default": true, diff --git a/ggsql-vscode/src/dataExplorer/filters.ts b/ggsql-vscode/src/dataExplorer/filters.ts new file mode 100644 index 000000000..f4755e0ad --- /dev/null +++ b/ggsql-vscode/src/dataExplorer/filters.ts @@ -0,0 +1,156 @@ +/* + * Positron's row filters, sort keys and column searches, in the kernel's terms. + */ + +import type { + ColumnFilter, + ColumnSortKey, + FilterBetween, + FilterComparison, + FilterMatchDataTypes, + FilterSetMembership, + FilterTextSearch, + KernelColumn, + KernelFilter, + KernelSortKey, + RowFilter, + SearchSchemaParams, +} from './protocol'; + +const TEXT_MATCH = { + contains: 'contains', + not_contains: 'not_contains', + starts_with: 'starts_with', + ends_with: 'ends_with', + regex_match: 'regex', +} as const; + +const COMPARE_OPS = new Set(['=', '!=', '<', '<=', '>', '>=']); + +/** + * A row filter as the kernel's filter, or the reason it cannot be one. + * Values are checked against the column's type by the kernel, not here. + */ +export function toKernelFilter(filter: RowFilter): KernelFilter | { error: string } { + const column = filter.column_schema?.column_index; + if (typeof column !== 'number') { + return { error: 'The filter names no column' }; + } + const params = filter.params ?? {}; + const missing = { error: 'The filter is missing a value' }; + + switch (filter.filter_type) { + case 'compare': { + const { op, value } = params as FilterComparison; + if (!COMPARE_OPS.has(op)) { + return { error: `Unknown comparison ${JSON.stringify(op)}` }; + } + if (typeof value !== 'string') { + return missing; + } + return { type: 'compare', column, op, value }; + } + case 'between': + case 'not_between': { + const { left_value: low, right_value: high } = params as FilterBetween; + if (typeof low !== 'string' || typeof high !== 'string') { + return missing; + } + return { type: 'between', column, low, high, negate: filter.filter_type === 'not_between' }; + } + case 'is_null': + case 'not_null': + return { type: 'is_null', column, negate: filter.filter_type === 'not_null' }; + case 'is_empty': + case 'not_empty': + return { type: 'is_empty', column, negate: filter.filter_type === 'not_empty' }; + case 'is_true': + return { type: 'is_true', column }; + case 'is_false': + return { type: 'is_false', column }; + case 'search': { + const { search_type, term, case_sensitive } = params as FilterTextSearch; + const match = TEXT_MATCH[search_type]; + if (!match) { + return { error: `Unknown search ${JSON.stringify(search_type)}` }; + } + if (typeof term !== 'string') { + return missing; + } + return { type: 'text', column, match, term, case_sensitive: !!case_sensitive }; + } + case 'set_membership': { + const { values, inclusive } = params as FilterSetMembership; + if (!Array.isArray(values)) { + return missing; + } + return { type: 'in', column, values, negate: inclusive === false }; + } + default: + return { error: `Unsupported filter type ${JSON.stringify(filter.filter_type)}` }; + } +} + +export function toKernelSortKeys(keys: ColumnSortKey[]): KernelSortKey[] { + return keys.map(k => ({ column: k.column_index, ascending: k.ascending })); +} + +/** Whether a column passes one of `search_schema`'s filters. */ +function columnMatches(filter: ColumnFilter, column: KernelColumn): boolean { + if (filter.filter_type === 'match_data_types') { + const { display_types } = filter.params as FilterMatchDataTypes; + return Array.isArray(display_types) && display_types.includes(column.kind); + } + if (filter.filter_type !== 'text_search') { + return true; + } + const { search_type, case_sensitive } = filter.params as FilterTextSearch; + const term = (filter.params as FilterTextSearch).term ?? ''; + if (search_type === 'regex_match') { + try { + return new RegExp(term, case_sensitive ? '' : 'i').test(column.name); + } catch { + return false; + } + } + const name = case_sensitive ? column.name : column.name.toLowerCase(); + const needle = case_sensitive ? term : term.toLowerCase(); + switch (search_type ?? 'contains') { + case 'contains': + return name.includes(needle); + case 'not_contains': + return !name.includes(needle); + case 'starts_with': + return name.startsWith(needle); + case 'ends_with': + return name.endsWith(needle); + default: + return true; + } +} + +function compare(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** The indices of the columns passing every filter, in the order asked for. */ +export function searchSchema(params: SearchSchemaParams, columns: KernelColumn[]): number[] { + const filters = params.filters ?? []; + const matches = columns + .map((_, i) => i) + .filter(i => filters.every(f => columnMatches(f, columns[i]))); + const name = (i: number) => columns[i].name.toLowerCase(); + const kind = (i: number) => columns[i].kind; + switch (params.sort_order) { + case 'ascending_name': + return matches.sort((a, b) => compare(name(a), name(b))); + case 'descending_name': + return matches.sort((a, b) => compare(name(b), name(a))); + case 'ascending_type': + return matches.sort((a, b) => compare(kind(a), kind(b))); + case 'descending_type': + return matches.sort((a, b) => compare(kind(b), kind(a))); + default: + return matches; + } +} diff --git a/ggsql-vscode/src/dataExplorer/format.ts b/ggsql-vscode/src/dataExplorer/format.ts new file mode 100644 index 000000000..327985cb0 --- /dev/null +++ b/ggsql-vscode/src/dataExplorer/format.ts @@ -0,0 +1,57 @@ +/* + * Cells from the kernel as Positron's `ColumnValue`s. + */ + +import type { ColumnKind, ColumnValue } from './protocol'; + +/** Positron's codes for values that are not shown as text. */ +export const SpecialValue = { + Null: 0, + NaN: 2, + Inf: 10, + NegInf: 11, +} as const; + +/** + * A kernel cell as Positron shows it: the text, or a special value's code. + * Only a floating column's text can spell a non-finite number; in any other + * column "NaN" is just text. + */ +export function cellValue(cell: string | null, kind: ColumnKind): ColumnValue { + if (cell === null) { + return SpecialValue.Null; + } + if (kind === 'floating') { + switch (cell.toLowerCase()) { + case 'nan': + return SpecialValue.NaN; + case 'inf': + case 'infinity': + return SpecialValue.Inf; + case '-inf': + case '-infinity': + return SpecialValue.NegInf; + } + } + return cell; +} + +/** `first..=last`, empty when `last < first`. */ +export function span(first: number, last: number): number[] { + const out: number[] = []; + for (let i = first; i <= last; i++) { + out.push(i); + } + return out; +} + +/** A thrown value's message: kernel errors arrive as plain objects. */ +export function errorMessage(err: unknown): string { + if (err instanceof Error) { + return err.message; + } + if (typeof err === 'object' && err !== null && typeof (err as { message?: unknown }).message === 'string') { + return (err as { message: string }).message; + } + return String(err); +} diff --git a/ggsql-vscode/src/dataExplorer/handler.ts b/ggsql-vscode/src/dataExplorer/handler.ts new file mode 100644 index 000000000..5561b6c1e --- /dev/null +++ b/ggsql-vscode/src/dataExplorer/handler.ts @@ -0,0 +1,503 @@ +/* + * The Data Explorer backend for console query results. + * + * Positron sends Data Explorer requests for the `ggsql` provider here, and + * each is answered by calling the kernel's results API (`result_*` methods, + * see `ggsql-jupyter/src/results/methods.rs`) through `callMethod` on the + * session that ran the query. A dataset id is `/`, and + * each open dataset holds one kernel view, opened on first use and closed + * with the Data Explorer. + */ + +import type * as positron from '@posit-dev/positron'; +import { cellValue, errorMessage, span } from './format'; +import { searchSchema, toKernelFilter, toKernelSortKeys } from './filters'; +import { addProfile, toProfileCalls } from './profiles'; +import type { + BackendState, + ColumnProfileResult, + ColumnSchema, + ColumnSelection, + ColumnSortKey, + ColumnValue, + ExportFormat, + GetColumnProfilesParams, + KernelFilter, + KernelProfile, + KernelRows, + OpenedResult, + RowFilter, + SearchSchemaParams, + TableSelection, +} from './protocol'; + +/** The provider id the backend registers under. */ +export const PROVIDER_ID = 'ggsql'; + +/** The part of a runtime session the backend uses. */ +export interface ResultsSession { + callMethod?(method: string, ...args: any[]): Thenable; +} + +/** A result's reference, `/`, as the console link names it. */ +export function resultRef(sessionId: string, key: string): string { + return `${sessionId}/${key}`; +} + +/** + * A result reference's session and key, or undefined when it is not one. + * Only the hex keys the kernel produces are accepted. + */ +export function parseResultRef(ref: string): { sessionId: string; key: string } | undefined { + const [sessionId, key, ...rest] = ref.split('/'); + if (!sessionId || !key || rest.length > 0 || !/^[0-9a-f]+$/.test(key)) { + return undefined; + } + return { sessionId, key }; +} + +/** + * The dataset id for result `key` of session `sessionId`: the reference + * behind a `ggsql:` scheme. Positron puts the id in the explorer editor's URI + * and parses it back out, accepting only a UUID or `:` + * (`PositronDataExplorerUri.parse`), so an id without a scheme opens an + * empty editor. + */ +export function datasetId(sessionId: string, key: string): string { + return `${PROVIDER_ID}:${resultRef(sessionId, key)}`; +} + +/** A dataset id's session and result key, or undefined when it is not one. */ +export function parseDatasetId(id: string): { sessionId: string; key: string } | undefined { + const prefix = `${PROVIDER_ID}:`; + return id.startsWith(prefix) ? parseResultRef(id.slice(prefix.length)) : undefined; +} + +/** One open dataset: its kernel view and what Positron last set on it. */ +export interface Dataset { + sessionId: string; + result: OpenedResult; + selectedRows: number; + rowFilters: RowFilter[]; + sortKeys: ColumnSortKey[]; +} + +function supported(field: string, types: T[]): Array> { + return types.map(t => ({ [field]: t, support_status: 'supported' })); +} + +const SUPPORTED_FEATURES = { + search_schema: { + support_status: 'supported', + supported_types: supported('column_filter_type', ['text_search', 'match_data_types']), + }, + set_column_filters: { + support_status: 'unsupported', + supported_types: [], + }, + set_row_filters: { + support_status: 'supported', + // The kernel ANDs every filter. + supports_conditions: 'unsupported', + supported_types: supported('row_filter_type', [ + 'between', 'compare', 'is_empty', 'is_false', 'is_null', 'is_true', + 'not_between', 'not_empty', 'not_null', 'search', 'set_membership', + ]), + }, + get_column_profiles: { + support_status: 'supported', + supported_types: supported('profile_type', [ + 'null_count', 'summary_stats', 'small_histogram', 'small_frequency_table', + ]), + }, + set_sort_columns: { + support_status: 'supported', + }, + export_data_selection: { + support_status: 'supported', + supported_formats: ['csv', 'tsv', 'html'], + }, + convert_to_code: { + support_status: 'supported', + code_syntaxes: [{ code_syntax_name: 'SQL' }], + }, +}; + +/** The state of a dataset whose result could not be opened. */ +function disconnectedState(id: string, message: string): BackendState { + const empty = { num_rows: 0, num_columns: 0 }; + return { + display_name: id, + connected: false, + error_message: message, + table_shape: empty, + table_unfiltered_shape: empty, + has_row_labels: false, + column_filters: [], + row_filters: [], + sort_keys: [], + supported_features: { + search_schema: { support_status: 'unsupported', supported_types: [] }, + set_column_filters: { support_status: 'unsupported', supported_types: [] }, + set_row_filters: { support_status: 'unsupported', supports_conditions: 'unsupported', supported_types: [] }, + get_column_profiles: { support_status: 'unsupported', supported_types: [] }, + set_sort_columns: { support_status: 'unsupported' }, + export_data_selection: { support_status: 'unsupported', supported_formats: [] }, + convert_to_code: { support_status: 'unsupported' }, + }, + }; +} + +export class ResultExplorer { + private readonly sessions = new Map(); + private readonly datasets = new Map>(); + + constructor( + private readonly sendUiEvent: (event: positron.DataExplorerUiEvent) => void, + private readonly log: (message: string) => void = () => { } + ) { } + + /** Make a session's results reachable. Dispose to forget the session. */ + addSession(sessionId: string, session: ResultsSession): { dispose(): void } { + this.sessions.set(sessionId, session); + return { + dispose: () => { + if (this.sessions.get(sessionId) === session) { + this.sessions.delete(sessionId); + } + this.forgetViews(sessionId); + }, + }; + } + + /** + * Forget the views of a session whose kernel has exited or restarted, + * since they went with it, and tell each open Data Explorer its schema + * changed. That makes it ask for its state again, which opens the result + * afresh: the explorer then shows the result, or the kernel's explanation + * of why it is gone, rather than a sort or filter that silently did nothing. + */ + forgetViews(sessionId: string): void { + for (const id of [...this.datasets.keys()]) { + if (parseDatasetId(id)?.sessionId === sessionId) { + this.datasets.delete(id); + this.sendUiEvent({ uri: id, method: 'schema_update', params: {} }); + } + } + } + + /** The dataset `id`, opening a kernel view of it the first time. */ + open(id: string): Promise { + let dataset = this.datasets.get(id); + if (!dataset) { + dataset = this.openView(id); + this.datasets.set(id, dataset); + // A failed open is not kept, so the next request tries again. + dataset.catch(() => { + if (this.datasets.get(id) === dataset) { + this.datasets.delete(id); + } + }); + } + return dataset; + } + + private async openView(id: string): Promise { + const parsed = parseDatasetId(id); + if (!parsed) { + throw new Error(`Not a ggsql result: ${id}`); + } + const result = await this.call(parsed.sessionId, 'result_open', { key: parsed.key }); + return { + sessionId: parsed.sessionId, + result, + selectedRows: result.num_rows, + rowFilters: [], + sortKeys: [], + }; + } + + /** Release the kernel view of a dataset whose Data Explorer closed. */ + closeDataset(id: string): void { + const dataset = this.datasets.get(id); + this.datasets.delete(id); + dataset + ?.then(ds => this.call(ds.sessionId, 'result_close', { view: ds.result.view })) + .catch(err => this.log(`Closing ${id} failed: ${errorMessage(err)}`)); + } + + private async call(sessionId: string, method: string, params: object): Promise { + const session = this.sessions.get(sessionId); + if (!session?.callMethod) { + throw new Error('The ggsql session that ran this query is no longer running.'); + } + return await session.callMethod(method, params) as T; + } + + async handleRpc(request: positron.DataExplorerRpcRequest): Promise { + try { + return { result: await this.dispatch(request) }; + } catch (err) { + return { error_message: errorMessage(err) }; + } + } + + private async dispatch(request: positron.DataExplorerRpcRequest): Promise { + const id = request.uri; + if (id === undefined) { + throw new Error(`No dataset given for ${request.method}`); + } + const params = request.params as any; + + if (request.method === 'get_state') { + let ds: Dataset; + try { + ds = await this.open(id); + } catch (err) { + return disconnectedState(id, errorMessage(err)); + } + return this.getState(ds); + } + + const ds = await this.open(id); + switch (request.method) { + case 'open_dataset': + return {}; + case 'get_schema': + return this.getSchema(ds, params.column_indices ?? []); + case 'search_schema': + return { matches: searchSchema(params as SearchSchemaParams, ds.result.columns) }; + case 'get_data_values': + return this.getDataValues(ds, params.columns ?? []); + case 'get_row_labels': + return { row_labels: [] }; + case 'set_row_filters': + return this.setRowFilters(ds, params.filters ?? []); + case 'set_sort_columns': + return this.setSortColumns(ds, params.sort_keys ?? []); + case 'set_column_filters': + // Unsupported, and advertised as such in `get_state`. + return null; + case 'get_column_profiles': + void this.sendColumnProfiles(id, ds, params as GetColumnProfilesParams); + return {}; + case 'export_data_selection': + return this.exportDataSelection(ds, params.selection, params.format ?? 'csv'); + case 'convert_to_code': { + const { sql } = await this.call<{ sql: string[] }>(ds.sessionId, 'result_sql', { view: ds.result.view }); + return { converted_code: sql }; + } + case 'suggest_code_syntax': + return { code_syntax_name: 'SQL' }; + default: + throw new Error(`Unknown method ${JSON.stringify(request.method)}`); + } + } + + private getState(ds: Dataset): BackendState { + const num_columns = ds.result.columns.length; + return { + display_name: ds.result.title, + table_shape: { num_rows: ds.selectedRows, num_columns }, + table_unfiltered_shape: { num_rows: ds.result.num_rows, num_columns }, + has_row_labels: false, + column_filters: [], + row_filters: ds.rowFilters, + sort_keys: ds.sortKeys, + supported_features: SUPPORTED_FEATURES, + }; + } + + private getSchema(ds: Dataset, indices: number[]): { columns: ColumnSchema[] } { + const columns: ColumnSchema[] = []; + for (const i of indices) { + const col = ds.result.columns[i]; + if (col) { + columns.push({ + column_name: col.name, + column_index: i, + type_name: col.type_name, + type_display: col.kind, + }); + } + } + return { columns }; + } + + /** Read cells; columns asking for the same rows share one `result_page`. */ + private async getDataValues(ds: Dataset, selections: ColumnSelection[]): Promise<{ columns: ColumnValue[][] }> { + const groups = new Map(); + selections.forEach((sel, slot) => { + const rows: KernelRows = 'indices' in sel.spec + ? { indices: sel.spec.indices } + : { first: sel.spec.first_index, last: sel.spec.last_index }; + const key = JSON.stringify(rows); + let group = groups.get(key); + if (!group) { + group = { rows, slots: [], columns: [] }; + groups.set(key, group); + } + group.slots.push(slot); + group.columns.push(sel.column_index); + }); + + const out: ColumnValue[][] = selections.map(() => []); + await Promise.all([...groups.values()].map(async group => { + const page = await this.call<{ columns: (string | null)[][] }>(ds.sessionId, 'result_page', { + view: ds.result.view, + rows: group.rows, + columns: group.columns, + }); + group.slots.forEach((slot, j) => { + const kind = ds.result.columns[group.columns[j]]?.kind ?? 'unknown'; + out[slot] = (page.columns[j] ?? []).map(cell => cellValue(cell, kind)); + }); + })); + return { columns: out }; + } + + private async setRowFilters(ds: Dataset, filters: RowFilter[]): Promise<{ selected_num_rows: number; had_errors: boolean }> { + const translated = filters.map(toKernelFilter); + const errors: (string | null)[] = translated.map(t => 'error' in t ? t.error : null); + const sent: KernelFilter[] = []; + const sentAt: number[] = []; + translated.forEach((t, i) => { + if (!('error' in t)) { + sent.push(t); + sentAt.push(i); + } + }); + + const reply = await this.call<{ num_rows: number; errors: (string | null)[] }>( + ds.sessionId, 'result_set_filters', { view: ds.result.view, filters: sent } + ); + sentAt.forEach((i, j) => { + errors[i] = reply.errors?.[j] ?? null; + }); + + ds.rowFilters = filters.map((filter, i) => { + const { error_message: _, ...rest } = filter; + const error = errors[i]; + return error === null + ? { ...rest, is_valid: true } + : { ...rest, is_valid: false, error_message: error }; + }); + ds.selectedRows = reply.num_rows; + return { selected_num_rows: reply.num_rows, had_errors: errors.some(e => e !== null) }; + } + + private async setSortColumns(ds: Dataset, keys: ColumnSortKey[]): Promise { + await this.call(ds.sessionId, 'result_set_sort', { + view: ds.result.view, + keys: toKernelSortKeys(keys), + }); + ds.sortKeys = keys; + return {}; + } + + /** Answer `get_column_profiles` with a `return_column_profiles` event. */ + private async sendColumnProfiles(id: string, ds: Dataset, params: GetColumnProfilesParams): Promise { + const profiles = await Promise.all((params.profiles ?? []).map(async request => { + const out: ColumnProfileResult = {}; + const kind = ds.result.columns[request.column_index]?.kind ?? 'unknown'; + for (const call of toProfileCalls(request.profiles ?? [])) { + try { + const profile = await this.call(ds.sessionId, 'result_profile', { + view: ds.result.view, + column: request.column_index, + profiles: call.request, + }); + addProfile(out, call, profile, kind); + } catch (err) { + this.log(`Profiling column ${request.column_index} of ${id} failed: ${errorMessage(err)}`); + } + } + return out; + })); + this.sendUiEvent({ + uri: id, + method: 'return_column_profiles', + params: { callback_id: params.callback_id, profiles }, + }); + } + + private async exportDataSelection( + ds: Dataset, + selection: TableSelection, + format: ExportFormat + ): Promise<{ data: string; format: ExportFormat }> { + const all = span(0, ds.result.columns.length - 1); + let rows: KernelRows | undefined; + let columns: number[]; + switch (selection.kind) { + case 'single_cell': { + // A single cell is copied as its bare value. + const { row_index, column_index } = selection.selection; + const page = await this.call<{ columns: (string | null)[][] }>(ds.sessionId, 'result_page', { + view: ds.result.view, + rows: { indices: [row_index] }, + columns: [column_index], + }); + return { data: page.columns[0]?.[0] ?? '', format }; + } + case 'cell_range': { + const s = selection.selection; + rows = { first: s.first_row_index, last: s.last_row_index }; + columns = span(s.first_column_index, s.last_column_index); + break; + } + case 'cell_indices': + rows = { indices: selection.selection.row_indices }; + columns = selection.selection.column_indices; + break; + case 'column_range': + columns = span(selection.selection.first_index, selection.selection.last_index); + break; + case 'row_range': + rows = { first: selection.selection.first_index, last: selection.selection.last_index }; + columns = all; + break; + case 'column_indices': + columns = selection.selection.indices; + break; + case 'row_indices': + rows = { indices: selection.selection.indices }; + columns = all; + break; + default: + throw new Error(`Unknown selection kind ${JSON.stringify((selection as { kind: unknown }).kind)}`); + } + const { data } = await this.call<{ data: string }>(ds.sessionId, 'result_export', { + view: ds.result.view, + ...(rows ? { rows } : {}), + columns, + format, + }); + return { data, format }; + } +} + +/** + * Register the backend with Positron. Returns undefined, registering nothing, + * when this Positron has no extension-hosted Data Explorer backends. + */ +export function registerResultExplorer( + api: positron.PositronApi, + subscriptions: { dispose(): unknown }[], + log: (message: string) => void +): ResultExplorer | undefined { + const dataExplorer = (api as Partial).dataExplorer; + if (!dataExplorer?.registerRpcHandler) { + return undefined; + } + // The handler is built before the registration it sends events through. + // eslint-disable-next-line prefer-const + let registration: positron.DataExplorerRpcSession | undefined; + const explorer = new ResultExplorer(event => registration?.sendUiEvent(event), log); + registration = dataExplorer.registerRpcHandler(PROVIDER_ID, { + handleRpc: request => explorer.handleRpc(request), + closeDataset: id => explorer.closeDataset(id), + }); + subscriptions.push(registration); + return explorer; +} diff --git a/ggsql-vscode/src/dataExplorer/profiles.ts b/ggsql-vscode/src/dataExplorer/profiles.ts new file mode 100644 index 000000000..bafdbec60 --- /dev/null +++ b/ggsql-vscode/src/dataExplorer/profiles.ts @@ -0,0 +1,152 @@ +/* + * Positron's column profile requests in the kernel's terms, and the kernel's + * profiles in Positron's `ColumnProfileResult` shape. + */ + +import type { + ColumnFrequencyTableParams, + ColumnHistogram, + ColumnHistogramParams, + ColumnKind, + ColumnProfileResult, + ColumnProfileSpec, + ColumnSummaryStats, + KernelProfile, + KernelProfileRequest, +} from './protocol'; + +/** Which Positron key each part of one kernel request answers. */ +export interface ProfileCall { + request: KernelProfileRequest; + histogramKey?: 'small_histogram' | 'large_histogram'; + frequencyKey?: 'small_frequency_table' | 'large_frequency_table'; +} + +/** + * The kernel requests for one column's specs. The kernel computes one + * histogram and one frequency table per call, so the large variants, if + * asked for, get a call of their own. + */ +export function toProfileCalls(specs: ColumnProfileSpec[]): ProfileCall[] { + const small: ProfileCall = { request: {} }; + const large: ProfileCall = { request: {} }; + for (const spec of specs) { + switch (spec.profile_type) { + case 'null_count': + small.request.null_count = true; + break; + case 'summary_stats': + small.request.summary = true; + break; + case 'small_histogram': + case 'large_histogram': { + const call = spec.profile_type === 'small_histogram' ? small : large; + const params = spec.params as ColumnHistogramParams | undefined; + call.request.histogram = { + bins: params?.num_bins ?? 20, + quantiles: params?.quantiles ?? [], + }; + call.histogramKey = spec.profile_type; + break; + } + case 'small_frequency_table': + case 'large_frequency_table': { + const call = spec.profile_type === 'small_frequency_table' ? small : large; + const params = spec.params as ColumnFrequencyTableParams | undefined; + call.request.frequency = { limit: params?.limit ?? 8 }; + call.frequencyKey = spec.profile_type; + break; + } + } + } + return [small, large].filter(c => Object.keys(c.request).length > 0); +} + +function str(v: unknown): string | undefined { + return typeof v === 'string' ? v : undefined; +} + +function num(v: unknown): number | undefined { + return typeof v === 'number' ? v : undefined; +} + +/** The kernel's summary of a column of `kind` as Positron's summary stats. */ +export function toSummaryStats(kind: ColumnKind, summary: Record): ColumnSummaryStats { + switch (kind) { + case 'integer': + case 'floating': + return { + type_display: kind, + number_stats: { + min_value: str(summary.min), + max_value: str(summary.max), + mean: str(summary.mean), + median: str(summary.median), + stdev: str(summary.stdev), + }, + }; + case 'boolean': + return { + type_display: kind, + boolean_stats: { + true_count: num(summary.true_count) ?? 0, + false_count: num(summary.false_count) ?? 0, + }, + }; + case 'string': + return { + type_display: kind, + string_stats: { + num_unique: num(summary.num_unique) ?? 0, + num_empty: num(summary.num_empty) ?? 0, + }, + }; + case 'date': + case 'datetime': { + const stats = { + min_date: str(summary.min), + max_date: str(summary.max), + num_unique: num(summary.num_unique), + }; + return kind === 'date' + ? { type_display: kind, date_stats: stats } + : { type_display: kind, datetime_stats: stats }; + } + default: + return { type_display: kind }; + } +} + +function toHistogram(h: NonNullable): ColumnHistogram { + return { + bin_edges: h.bin_edges, + bin_counts: h.bin_counts, + // The kernel's quantiles are computed in SQL over every row, so exact. + quantiles: h.quantiles.map(({ q, value }) => ({ q, value, exact: true })), + }; +} + +/** Merge one kernel profile into the Positron result, under the keys `call` asked for. */ +export function addProfile( + out: ColumnProfileResult, + call: ProfileCall, + profile: KernelProfile, + kind: ColumnKind +): void { + if (profile.null_count !== undefined) { + out.null_count = profile.null_count; + } + if (profile.summary !== undefined) { + out.summary_stats = toSummaryStats(kind, profile.summary); + } + if (profile.histogram && call.histogramKey) { + out[call.histogramKey] = toHistogram(profile.histogram); + } + if (profile.frequency && call.frequencyKey) { + out[call.frequencyKey] = { + values: profile.frequency.values, + counts: profile.frequency.counts, + other_count: profile.frequency.other_count, + }; + } +} diff --git a/ggsql-vscode/src/dataExplorer/protocol.ts b/ggsql-vscode/src/dataExplorer/protocol.ts new file mode 100644 index 000000000..210960cac --- /dev/null +++ b/ggsql-vscode/src/dataExplorer/protocol.ts @@ -0,0 +1,267 @@ +/* + * The two protocols the Data Explorer backend translates between. + * + * Positron's half is a hand-written subset of its Data Explorer backend + * protocol (`positronDataExplorerComm.ts` in Positron): only the requests, + * results and events this backend answers, with the fields it reads or + * writes. The kernel's half is the frontend-neutral results API that + * `ggsql-jupyter/src/results/methods.rs` answers. + */ + +// --- Positron's Data Explorer protocol --- + +/** A column's display type. Every kernel `kind` is one of these. */ +export type ColumnDisplayType = + | 'boolean' | 'string' | 'date' | 'datetime' | 'time' | 'interval' | 'object' + | 'array' | 'struct' | 'unknown' | 'floating' | 'integer' | 'decimal'; + +export type SupportStatus = 'supported' | 'unsupported'; + +export interface TableShape { + num_rows: number; + num_columns: number; +} + +export interface ColumnSchema { + column_name: string; + column_index: number; + type_name: string; + type_display: ColumnDisplayType; +} + +export type RowFilterType = + | 'between' | 'compare' | 'is_empty' | 'is_false' | 'is_null' | 'is_true' + | 'not_between' | 'not_empty' | 'not_null' | 'search' | 'set_membership'; + +export type TextSearchType = 'contains' | 'not_contains' | 'starts_with' | 'ends_with' | 'regex_match'; + +export interface FilterBetween { + left_value: string; + right_value: string; +} + +export interface FilterComparison { + op: '=' | '!=' | '<' | '<=' | '>' | '>='; + value: string; +} + +export interface FilterSetMembership { + values: string[]; + inclusive: boolean; +} + +export interface FilterTextSearch { + search_type: TextSearchType; + term: string; + case_sensitive: boolean; +} + +export interface FilterMatchDataTypes { + display_types: ColumnDisplayType[]; +} + +export interface RowFilter { + filter_id: string; + filter_type: RowFilterType; + column_schema: ColumnSchema; + condition: 'and' | 'or'; + is_valid?: boolean; + error_message?: string; + params?: FilterBetween | FilterComparison | FilterTextSearch | FilterSetMembership; +} + +export interface ColumnFilter { + filter_type: 'text_search' | 'match_data_types'; + params: FilterTextSearch | FilterMatchDataTypes; +} + +export interface ColumnSortKey { + column_index: number; + ascending: boolean; +} + +export type SearchSchemaSortOrder = + | 'original' | 'ascending_name' | 'descending_name' | 'ascending_type' | 'descending_type'; + +export interface SearchSchemaParams { + filters: ColumnFilter[]; + sort_order: SearchSchemaSortOrder; +} + +export interface DataSelectionSingleCell { + row_index: number; + column_index: number; +} + +export interface DataSelectionCellRange { + first_row_index: number; + last_row_index: number; + first_column_index: number; + last_column_index: number; +} + +export interface DataSelectionCellIndices { + row_indices: number[]; + column_indices: number[]; +} + +export interface DataSelectionRange { + first_index: number; + last_index: number; +} + +export interface DataSelectionIndices { + indices: number[]; +} + +export type ArraySelection = DataSelectionRange | DataSelectionIndices; + +export interface ColumnSelection { + column_index: number; + spec: ArraySelection; +} + +export type TableSelection = + | { kind: 'single_cell'; selection: DataSelectionSingleCell } + | { kind: 'cell_range'; selection: DataSelectionCellRange } + | { kind: 'cell_indices'; selection: DataSelectionCellIndices } + | { kind: 'column_range' | 'row_range'; selection: DataSelectionRange } + | { kind: 'column_indices' | 'row_indices'; selection: DataSelectionIndices }; + +export type ExportFormat = 'csv' | 'tsv' | 'html'; + +/** A cell: formatted text, or a code for a special value (see `format.ts`). */ +export type ColumnValue = number | string; + +export type ColumnProfileType = + | 'null_count' | 'summary_stats' | 'small_frequency_table' | 'large_frequency_table' + | 'small_histogram' | 'large_histogram'; + +export interface ColumnHistogramParams { + method: string; + num_bins: number; + quantiles?: number[]; +} + +export interface ColumnFrequencyTableParams { + limit: number; +} + +export interface ColumnProfileSpec { + profile_type: ColumnProfileType; + params?: ColumnHistogramParams | ColumnFrequencyTableParams; +} + +export interface ColumnProfileRequest { + column_index: number; + profiles: ColumnProfileSpec[]; +} + +export interface GetColumnProfilesParams { + callback_id: string; + profiles: ColumnProfileRequest[]; +} + +export interface ColumnHistogram { + bin_edges: string[]; + bin_counts: number[]; + quantiles: { q: number; value: string; exact: boolean }[]; +} + +export interface ColumnFrequencyTable { + values: ColumnValue[]; + counts: number[]; + other_count?: number; +} + +export interface ColumnSummaryStats { + type_display: ColumnDisplayType; + number_stats?: { min_value?: string; max_value?: string; mean?: string; median?: string; stdev?: string }; + string_stats?: { num_empty: number; num_unique: number }; + boolean_stats?: { true_count: number; false_count: number }; + date_stats?: { num_unique?: number; min_date?: string; max_date?: string }; + datetime_stats?: { num_unique?: number; min_date?: string; max_date?: string }; +} + +export interface ColumnProfileResult { + null_count?: number; + summary_stats?: ColumnSummaryStats; + small_histogram?: ColumnHistogram; + large_histogram?: ColumnHistogram; + small_frequency_table?: ColumnFrequencyTable; + large_frequency_table?: ColumnFrequencyTable; +} + +export interface BackendState { + display_name: string; + table_shape: TableShape; + table_unfiltered_shape: TableShape; + has_row_labels: boolean; + column_filters: ColumnFilter[]; + row_filters: RowFilter[]; + sort_keys: ColumnSortKey[]; + supported_features: unknown; + connected?: boolean; + error_message?: string; +} + +// --- The kernel's results API --- + +export type ColumnKind = + | 'integer' | 'floating' | 'boolean' | 'string' | 'date' | 'datetime' | 'time' | 'unknown'; + +export interface KernelColumn { + name: string; + type_name: string; + kind: ColumnKind; +} + +/** `result_open`'s result. */ +export interface OpenedResult { + view: string; + key: string; + title: string; + query: string; + columns: KernelColumn[]; + num_rows: number; +} + +/** Rows of a view: `first..=last`, or these positions in this order. */ +export type KernelRows = { first: number; last: number } | { indices: number[] }; + +export type KernelFilter = + | { type: 'compare'; column: number; op: FilterComparison['op']; value: string } + | { type: 'between'; column: number; low: string; high: string; negate: boolean } + | { type: 'is_null'; column: number; negate: boolean } + | { type: 'is_empty'; column: number; negate: boolean } + | { type: 'is_true'; column: number } + | { type: 'is_false'; column: number } + | { + type: 'text'; + column: number; + match: 'contains' | 'not_contains' | 'starts_with' | 'ends_with' | 'regex'; + term: string; + case_sensitive: boolean; + } + | { type: 'in'; column: number; values: string[]; negate: boolean }; + +export interface KernelSortKey { + column: number; + ascending: boolean; +} + +/** `result_profile`'s `profiles` parameter. */ +export interface KernelProfileRequest { + null_count?: boolean; + summary?: boolean; + histogram?: { bins: number; quantiles: number[] }; + frequency?: { limit: number }; +} + +/** `result_profile`'s result. `summary`'s fields depend on the column's kind. */ +export interface KernelProfile { + null_count?: number; + summary?: Record; + histogram?: { bin_edges: string[]; bin_counts: number[]; quantiles: { q: number; value: string }[] }; + frequency?: { values: string[]; counts: number[]; other_count: number }; +} diff --git a/ggsql-vscode/src/extension.ts b/ggsql-vscode/src/extension.ts index ba1d784cc..d61ab1680 100644 --- a/ggsql-vscode/src/extension.ts +++ b/ggsql-vscode/src/extension.ts @@ -17,6 +17,7 @@ import { CELL_LANGUAGE_IDS, isGgsqlDocument } from './languages'; import { activateSqlAssociationPrompt } from './sqlAssociation'; import { ggsqlDataImporter } from './dataImporter'; import * as path from 'path'; +import { registerResultExplorer } from './dataExplorer/handler'; // Output channel for logging const outputChannel = vscode.window.createOutputChannel('ggsql'); @@ -61,8 +62,15 @@ export function activate(context: vscode.ExtensionContext): void { log('Positron API acquired - registering runtime manager'); + // Serve console results to the Data Explorer, where this Positron has + // extension-hosted Data Explorer backends. + const resultExplorer = registerResultExplorer(positronApi, context.subscriptions, log); + log(resultExplorer + ? 'Data Explorer backend registered' + : 'positron.dataExplorer not available - console results will not link to the Data Explorer'); + // Running in Positron - register the ggsql runtime manager - const manager = new GgsqlRuntimeManager(context); + const manager = new GgsqlRuntimeManager(context, { resultExplorer }); const disposable = positronApi.runtime.registerLanguageRuntimeManager('ggsql', manager); context.subscriptions.push(disposable); diff --git a/ggsql-vscode/src/manager.ts b/ggsql-vscode/src/manager.ts index 83585c910..d3c430fdc 100644 --- a/ggsql-vscode/src/manager.ts +++ b/ggsql-vscode/src/manager.ts @@ -13,6 +13,8 @@ import * as crypto from 'crypto'; import type * as positron from '@posit-dev/positron'; import type { JupyterKernelSpec, PositronSupervisorApi } from './types'; import { log, showLog } from './extension'; +import { withResultLinks } from './resultLinks'; +import type { ResultExplorer } from './dataExplorer/handler'; /** Where a kernel candidate was discovered */ type KernelSource = 'Bundled' | 'Setting' | 'Jupyter' | 'System' | 'Path'; @@ -534,6 +536,19 @@ export function generateMetadata( }; } +/** + * The kernel environment for the `ggsql.consoleRows` setting. + * + * An environment variable rather than a flag: a kernel that predates + * `--console-rows` ignores the variable, where it would refuse to start on the + * flag. A value the kernel would reject is left out, so it uses its default. + */ +export function consoleRowsEnv(rows: unknown): Record { + return Number.isInteger(rows) && (rows as number) >= 1 + ? { GGSQL_CONSOLE_ROWS: String(rows) } + : {}; +} + /** * Create a Jupyter kernel spec for ggsql-jupyter * @@ -566,7 +581,10 @@ function createKernelSpec( display_name: 'ggsql', language: 'ggsql', interrupt_mode: 'signal', - env: { RUST_LOG: 'error' }, + env: { + RUST_LOG: 'error', + ...consoleRowsEnv(vscode.workspace.getConfiguration('ggsql').get('consoleRows')), + }, kernel_protocol_version: '5.3', }; } @@ -691,6 +709,12 @@ export async function getSupervisorApi(): Promise { * Overrides for GgsqlRuntimeManager's environment. */ export interface RuntimeManagerOptions { + /** + * The Data Explorer backend for console results. Sessions are wrapped to + * link their results to it; without one they are returned unwrapped. + */ + resultExplorer?: ResultExplorer; + /** * Directory the discovered kernel is advertised in, as a Jupyter kernel * spec. Defaults to the user-level Jupyter kernels directory. @@ -726,9 +750,11 @@ export class GgsqlRuntimeManager implements positron.LanguageRuntimeManager { private _context: vscode.ExtensionContext; private _kernelSpecDir: string; private _probe: KernelProbe; + private _resultExplorer: ResultExplorer | undefined; constructor(context: vscode.ExtensionContext, options: RuntimeManagerOptions = {}) { this._context = context; + this._resultExplorer = options.resultExplorer; this._kernelSpecDir = options.kernelSpecDir ?? getUserJupyterKernelDir(); this._probe = options.probe ?? probeKernel; } @@ -829,6 +855,10 @@ export class GgsqlRuntimeManager implements positron.LanguageRuntimeManager { return undefined; } + private _withResultLinks(session: positron.LanguageRuntimeSession): positron.LanguageRuntimeSession { + return this._resultExplorer ? withResultLinks(session, this._resultExplorer) : session; + } + /** * Create a new ggsql runtime session. */ @@ -858,7 +888,7 @@ export class GgsqlRuntimeManager implements positron.LanguageRuntimeManager { dynState ); - return session; + return this._withResultLinks(session); } /** @@ -885,7 +915,7 @@ export class GgsqlRuntimeManager implements positron.LanguageRuntimeManager { dynState ); - return session; + return this._withResultLinks(session); } /** diff --git a/ggsql-vscode/src/resultLinks.ts b/ggsql-vscode/src/resultLinks.ts new file mode 100644 index 000000000..ffdf32a57 --- /dev/null +++ b/ggsql-vscode/src/resultLinks.ts @@ -0,0 +1,150 @@ +/* + * "Open in Data Explorer" links under console query results. + * + * In a Positron console the kernel marks each tabular `execute_result` with + * `{"ggsql": {"result": ""}}` in its metadata, which the supervisor + * passes on as the Result message's `outputMetadata`. Each ggsql session is + * wrapped so that, after such a message, its console also receives a line of + * stdout holding an OSC 8 hyperlink to `x-ggsql-result:/`. + * + * Positron offers a clicked hyperlink to the last active console session of + * each language via `openResource`, which the supervisor's session does not + * implement, so the wrapper adds it. Handling the link opens a kernel view of + * the result through the Data Explorer backend (`dataExplorer/handler.ts`) + * and shows it with `positron.dataExplorer.open`. + */ + +import * as vscode from 'vscode'; +import type * as positron from '@posit-dev/positron'; +import { getPositronApi } from './positronApi'; +import { log } from './extension'; +import { PROVIDER_ID, ResultExplorer, datasetId, parseResultRef, resultRef } from './dataExplorer/handler'; +import { errorMessage } from './dataExplorer/format'; + +export const RESULT_SCHEME = 'x-ggsql-result'; + +/** A link's target, or undefined when the resource is not one of ours. */ +export function parseResultLink(resource: vscode.Uri | string): { sessionId: string; key: string } | undefined { + const uri = typeof resource === 'string' ? vscode.Uri.parse(resource) : resource; + if (uri.scheme !== RESULT_SCHEME) { + return undefined; + } + return parseResultRef(uri.path); +} + +/** The result key a message carries, when it is a Result marked by the kernel. */ +export function resultKey(message: positron.LanguageRuntimeMessage): string | undefined { + if (message.type !== ('result' as positron.LanguageRuntimeMessageType)) { + return undefined; + } + const metadata = (message as positron.LanguageRuntimeResult).outputMetadata; + const ggsql = metadata?.ggsql as { result?: unknown } | undefined; + const key = ggsql?.result; + return typeof key === 'string' && /^[0-9a-f]+$/.test(key) ? key : undefined; +} + +/** The stdout line, under a result, that links to it. */ +export function linkMessage( + result: positron.LanguageRuntimeMessage, + sessionId: string, + key: string +): positron.LanguageRuntimeStream { + const target = `${RESULT_SCHEME}:${resultRef(sessionId, key)}`; + return { + id: `${result.id}-ggsql-result-link`, + parent_id: result.parent_id, + when: result.when, + type: 'stream' as positron.LanguageRuntimeMessageType, + name: 'stdout' as positron.LanguageRuntimeStreamName, + text: `\x1b]8;;${target}\x1b\\Open in Data Explorer\x1b]8;;\x1b\\\n`, + }; +} + +/** Open a result in the Data Explorer. */ +async function openResult(explorer: ResultExplorer, sessionId: string, key: string): Promise { + const api = getPositronApi(); + if (!api) { + return; + } + const id = datasetId(sessionId, key); + try { + // Opened here rather than on the Data Explorer's first request, for + // the title, and so an expired result is reported before a tab opens. + const dataset = await explorer.open(id); + await api.dataExplorer.open({ + providerId: PROVIDER_ID, + datasetId: id, + displayName: dataset.result.title, + }); + } catch (err) { + const message = errorMessage(err); + log(`Opening result ${key} in session ${sessionId} failed: ${message}`); + vscode.window.showWarningMessage( + message || 'This query result could not be opened in the Data Explorer.' + ); + } +} + +/** + * Wrap a supervisor session to add result links to its console and handle + * `x-ggsql-result:` links. + * + * Every runtime message is re-fired in order, with the link line following + * each marked result in a console session. Everything else passes through to + * the session itself, with methods bound to it so its own `this` is never the + * proxy. + */ +export function withResultLinks( + session: positron.LanguageRuntimeSession, + explorer: ResultExplorer +): positron.LanguageRuntimeSession { + const sessionId = session.metadata.sessionId; + const isConsole = session.metadata.sessionMode === ('console' as positron.LanguageRuntimeSessionMode); + + const messages = new vscode.EventEmitter(); + const disposables: { dispose(): unknown }[] = [ + messages, + explorer.addSession(sessionId, session), + session.onDidReceiveRuntimeMessage(message => { + messages.fire(message); + const key = isConsole ? resultKey(message) : undefined; + if (key) { + messages.fire(linkMessage(message, sessionId, key)); + } + }), + // The kernel's views end with it. + session.onDidEndSession(() => explorer.forgetViews(sessionId)), + ]; + + const openResource = async (resource: vscode.Uri | string): Promise => { + const link = parseResultLink(resource); + if (!link) { + return false; + } + // Positron offers the link only to the most recently active ggsql + // console, which need not be the one that printed it; the session id + // routes it back to the right one. + void openResult(explorer, link.sessionId, link.key); + return true; + }; + + const dispose = () => { + disposables.forEach(d => d.dispose()); + return session.dispose(); + }; + + return new Proxy(session, { + get(target, prop) { + switch (prop) { + case 'openResource': + return openResource; + case 'onDidReceiveRuntimeMessage': + return messages.event; + case 'dispose': + return dispose; + } + const value = Reflect.get(target, prop, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); +} diff --git a/ggsql-vscode/src/test/dataExplorer.test.ts b/ggsql-vscode/src/test/dataExplorer.test.ts new file mode 100644 index 000000000..128fd7798 --- /dev/null +++ b/ggsql-vscode/src/test/dataExplorer.test.ts @@ -0,0 +1,390 @@ +import * as assert from 'assert'; +import type * as positron from '@posit-dev/positron'; +import { ResultExplorer, datasetId, parseDatasetId } from '../dataExplorer/handler'; +import { searchSchema, toKernelFilter } from '../dataExplorer/filters'; +import { toProfileCalls, toSummaryStats } from '../dataExplorer/profiles'; +import { cellValue } from '../dataExplorer/format'; +import type { KernelColumn, OpenedResult, RowFilter } from '../dataExplorer/protocol'; + +const COLUMNS: KernelColumn[] = [ + { name: 'n', type_name: 'BIGINT', kind: 'integer' }, + { name: 'x', type_name: 'DOUBLE', kind: 'floating' }, + { name: 'Label', type_name: 'VARCHAR', kind: 'string' }, +]; + +const OPENED: OpenedResult = { + view: 'v1', + key: '0f9e', + title: 'Query [3]', + query: 'SELECT 1', + columns: COLUMNS, + num_rows: 100, +}; + +const ID = 'ggsql:ggsql-abc/0f9e'; + +/** A results API stand-in that records each call and answers from `replies`. */ +function setup(replies: Record) => unknown> = {}) { + const calls: Array<[string, Record]> = []; + const events: positron.DataExplorerUiEvent[] = []; + const explorer = new ResultExplorer(e => events.push(e)); + explorer.addSession('ggsql-abc', { + callMethod: async (method: string, params: Record) => { + calls.push([method, params]); + if (method === 'result_open') { + return OPENED; + } + const reply = replies[method]; + if (!reply) { + throw { code: -32000, message: `no reply for ${method}` }; + } + return reply(params); + }, + }); + const rpc = (method: string, params: object = {}) => + explorer.handleRpc({ method, uri: ID, params }); + return { explorer, calls, events, rpc }; +} + +function filter(filter_type: string, column_index: number, params?: object): RowFilter { + return { + filter_id: `f-${filter_type}`, + filter_type: filter_type as RowFilter['filter_type'], + column_schema: { column_name: '', column_index, type_name: '', type_display: 'integer' }, + condition: 'and', + params: params as RowFilter['params'], + }; +} + +suite('Data Explorer backend', () => { + test('dataset ids are a session id and a hex key', () => { + assert.deepStrictEqual(parseDatasetId('ggsql:s/0a'), { sessionId: 's', key: '0a' }); + assert.strictEqual(parseDatasetId('s/0a'), undefined); + assert.strictEqual(parseDatasetId('ggsql:s/0A'), undefined); + assert.strictEqual(parseDatasetId('ggsql:s/0a/1'), undefined); + assert.strictEqual(datasetId('ggsql-abc', '0f9e'), ID); + // What Positron's PositronDataExplorerUri.parse accepts besides a UUID. + assert.match(datasetId('ggsql-abc', '0f9e'), /^[a-z][a-z0-9+.-]*:.+$/); + }); + + test('opens one kernel view per dataset, lazily', async () => { + const { calls, rpc } = setup(); + const state = await rpc('get_state'); + await rpc('get_schema', { column_indices: [2, 0, 9] }); + assert.deepStrictEqual(calls, [['result_open', { key: '0f9e' }]]); + const s = state.result as Record; + assert.strictEqual(s.display_name, 'Query [3]'); + assert.deepStrictEqual(s.table_shape, { num_rows: 100, num_columns: 3 }); + assert.strictEqual(s.supported_features.set_row_filters.supports_conditions, 'unsupported'); + assert.strictEqual(s.supported_features.set_column_filters.support_status, 'unsupported'); + }); + + test('get_schema describes the asked-for columns', async () => { + const { rpc } = setup(); + const r = await rpc('get_schema', { column_indices: [2, 9] }); + assert.deepStrictEqual(r.result, { + columns: [{ column_name: 'Label', column_index: 2, type_name: 'VARCHAR', type_display: 'string' }], + }); + }); + + test('a result that cannot be opened is a disconnected state', async () => { + const explorer = new ResultExplorer(() => { }); + explorer.addSession('ggsql-abc', { + callMethod: async () => { throw { message: 'This query result is no longer available.' }; }, + }); + const r = await explorer.handleRpc({ method: 'get_state', uri: ID, params: {} }); + const s = r.result as Record; + assert.strictEqual(s.connected, false); + assert.strictEqual(s.error_message, 'This query result is no longer available.'); + const page = await explorer.handleRpc({ method: 'get_schema', uri: ID, params: { column_indices: [0] } }); + assert.strictEqual(page.error_message, 'This query result is no longer available.'); + }); + + test('forgetting a session tells its open explorers to ask for their state again', async () => { + const events: { uri?: string; method: string }[] = []; + const explorer = new ResultExplorer(e => events.push(e as { uri?: string; method: string })); + let open = 0; + explorer.addSession('ggsql-abc', { + callMethod: async () => { + open += 1; + return { view: `v${open}`, key: '0f9e', title: 'Query [1]', query: '', columns: [], num_rows: 1 }; + }, + }); + await explorer.handleRpc({ method: 'get_state', uri: ID, params: {} }); + explorer.forgetViews('ggsql-abc'); + explorer.forgetViews('ggsql-other'); + assert.deepStrictEqual(events, [{ uri: ID, method: 'schema_update', params: {} }]); + // The next request opens a fresh view rather than reusing the old one. + await explorer.handleRpc({ method: 'get_state', uri: ID, params: {} }); + assert.strictEqual(open, 2); + }); + + test('a session that is gone is an error, not a hang', async () => { + const explorer = new ResultExplorer(() => { }); + const r = await explorer.handleRpc({ method: 'get_schema', uri: ID, params: { column_indices: [0] } }); + assert.match(r.error_message ?? '', /no longer running/); + }); + + test('get_data_values batches columns that ask for the same rows', async () => { + const { calls, rpc } = setup({ + result_page: p => ({ + columns: (p.columns as number[]).map(c => + c === 1 ? ['1.5', 'NaN', null] : c === 2 ? ['NaN', 'inf', null] : ['a']), + }), + }); + const r = await rpc('get_data_values', { + columns: [ + { column_index: 1, spec: { first_index: 0, last_index: 2 } }, + { column_index: 0, spec: { indices: [5] } }, + { column_index: 2, spec: { first_index: 0, last_index: 2 } }, + ], + format_options: {}, + }); + const pages = calls.filter(c => c[0] === 'result_page').map(c => c[1]); + assert.deepStrictEqual(pages, [ + { view: 'v1', rows: { first: 0, last: 2 }, columns: [1, 2] }, + { view: 'v1', rows: { indices: [5] }, columns: [0] }, + ]); + // NaN is special only in a floating column. + assert.deepStrictEqual(r.result, { columns: [['1.5', 2, 0], ['a'], ['NaN', 'inf', 0]] }); + }); + + test('set_row_filters translates, and marks the filters the kernel rejects', async () => { + const { calls, rpc } = setup({ + result_set_filters: () => ({ num_rows: 7, errors: [null, 'Not a number: "x"'] }), + result_page: () => ({ columns: [] }), + }); + const filters = [ + filter('compare', 0, { op: '>', value: '1' }), + filter('nonsense', 0), + { ...filter('between', 1, { left_value: 'x', right_value: '2' }), error_message: 'stale' }, + ]; + const r = await rpc('set_row_filters', { filters }); + assert.deepStrictEqual(r.result, { selected_num_rows: 7, had_errors: true }); + assert.deepStrictEqual(calls.at(-1), ['result_set_filters', { + view: 'v1', + filters: [ + { type: 'compare', column: 0, op: '>', value: '1' }, + { type: 'between', column: 1, low: 'x', high: '2', negate: false }, + ], + }]); + + const state = (await rpc('get_state')).result as Record; + assert.deepStrictEqual(state.table_shape, { num_rows: 7, num_columns: 3 }); + assert.deepStrictEqual( + state.row_filters.map((f: RowFilter) => [f.filter_id, f.is_valid, f.error_message]), + [ + ['f-compare', true, undefined], + ['f-nonsense', false, 'Unsupported filter type "nonsense"'], + ['f-between', false, 'Not a number: "x"'], + ] + ); + }); + + test('set_sort_columns passes the keys on and keeps them for get_state', async () => { + const { calls, rpc } = setup({ result_set_sort: () => ({}) }); + const sort_keys = [{ column_index: 2, ascending: false }]; + assert.deepStrictEqual((await rpc('set_sort_columns', { sort_keys })).result, {}); + assert.deepStrictEqual(calls.at(-1), ['result_set_sort', { view: 'v1', keys: [{ column: 2, ascending: false }] }]); + const state = (await rpc('get_state')).result as Record; + assert.deepStrictEqual(state.sort_keys, sort_keys); + }); + + test('a kernel error is the error_message', async () => { + const { rpc } = setup({ result_set_sort: () => { throw { message: 'There is no column 9' }; } }); + const r = await rpc('set_sort_columns', { sort_keys: [{ column_index: 9, ascending: true }] }); + assert.strictEqual(r.error_message, 'There is no column 9'); + }); + + test('get_column_profiles replies at once, then sends the profiles', async () => { + const { calls, events, rpc } = setup({ + result_profile: p => (p.column === 0 + ? { + null_count: 2, + summary: { min: '1', max: '9', mean: '5', median: '5', stdev: '2' }, + histogram: { bin_edges: ['1', '9'], bin_counts: [98], quantiles: [{ q: 0.5, value: '5' }] }, + } + : { frequency: { values: ['a'], counts: [3], other_count: 4 } }), + }); + const r = await rpc('get_column_profiles', { + callback_id: 'cb', + profiles: [ + { + column_index: 0, profiles: [ + { profile_type: 'null_count' }, + { profile_type: 'summary_stats' }, + { profile_type: 'small_histogram', params: { method: 'fixed', num_bins: 10, quantiles: [0.5] } }, + ], + }, + { column_index: 2, profiles: [{ profile_type: 'small_frequency_table', params: { limit: 5 } }] }, + ], + format_options: {}, + }); + assert.deepStrictEqual(r.result, {}); + await new Promise(resolve => setTimeout(resolve, 0)); + + assert.deepStrictEqual(calls.filter(c => c[0] === 'result_profile').map(c => c[1]), [ + { view: 'v1', column: 0, profiles: { null_count: true, summary: true, histogram: { bins: 10, quantiles: [0.5] } } }, + { view: 'v1', column: 2, profiles: { frequency: { limit: 5 } } }, + ]); + assert.deepStrictEqual(events, [{ + uri: ID, + method: 'return_column_profiles', + params: { + callback_id: 'cb', + profiles: [ + { + null_count: 2, + summary_stats: { + type_display: 'integer', + number_stats: { min_value: '1', max_value: '9', mean: '5', median: '5', stdev: '2' }, + }, + small_histogram: { bin_edges: ['1', '9'], bin_counts: [98], quantiles: [{ q: 0.5, value: '5', exact: true }] }, + }, + { small_frequency_table: { values: ['a'], counts: [3], other_count: 4 } }, + ], + }, + }]); + }); + + test('export_data_selection covers every selection kind', async () => { + const { calls, rpc } = setup({ + result_export: () => ({ data: 'csv' }), + result_page: () => ({ columns: [['42']] }), + }); + const cases: Array<[object, object | undefined, number[]]> = [ + [{ kind: 'cell_range', selection: { first_row_index: 1, last_row_index: 2, first_column_index: 0, last_column_index: 1 } }, { first: 1, last: 2 }, [0, 1]], + [{ kind: 'cell_indices', selection: { row_indices: [4, 1], column_indices: [2] } }, { indices: [4, 1] }, [2]], + [{ kind: 'column_range', selection: { first_index: 1, last_index: 2 } }, undefined, [1, 2]], + [{ kind: 'row_range', selection: { first_index: 3, last_index: 4 } }, { first: 3, last: 4 }, [0, 1, 2]], + [{ kind: 'column_indices', selection: { indices: [2, 0] } }, undefined, [2, 0]], + [{ kind: 'row_indices', selection: { indices: [7] } }, { indices: [7] }, [0, 1, 2]], + ]; + for (const [selection, rows, columns] of cases) { + const r = await rpc('export_data_selection', { selection, format: 'tsv' }); + assert.deepStrictEqual(r.result, { data: 'csv', format: 'tsv' }); + const expected: Record = { view: 'v1', columns, format: 'tsv' }; + if (rows) { + expected.rows = rows; + } + assert.deepStrictEqual(calls.at(-1), ['result_export', expected], JSON.stringify(selection)); + } + + const cell = await rpc('export_data_selection', { + selection: { kind: 'single_cell', selection: { row_index: 3, column_index: 1 } }, + format: 'csv', + }); + assert.deepStrictEqual(cell.result, { data: '42', format: 'csv' }); + assert.deepStrictEqual(calls.at(-1), ['result_page', { view: 'v1', rows: { indices: [3] }, columns: [1] }]); + }); + + test('convert_to_code is the kernel SQL', async () => { + const { rpc } = setup({ result_sql: () => ({ sql: ['SELECT *', 'FROM t'] }) }); + const r = await rpc('convert_to_code', { code_syntax_name: { code_syntax_name: 'SQL' } }); + assert.deepStrictEqual(r.result, { converted_code: ['SELECT *', 'FROM t'] }); + assert.deepStrictEqual((await rpc('suggest_code_syntax')).result, { code_syntax_name: 'SQL' }); + }); + + test('the unsupported requests answer emptily, and unknown ones are errors', async () => { + const { rpc } = setup(); + assert.deepStrictEqual((await rpc('get_row_labels', {})).result, { row_labels: [] }); + assert.strictEqual((await rpc('set_column_filters', { filters: [] })).result, null); + assert.match((await rpc('nope')).error_message ?? '', /Unknown method/); + }); + + test('closing a dataset closes its view, and the next request reopens', async () => { + const { explorer, calls, rpc } = setup({ result_close: () => ({}) }); + await rpc('get_state'); + explorer.closeDataset(ID); + await new Promise(resolve => setTimeout(resolve, 0)); + assert.deepStrictEqual(calls.at(-1), ['result_close', { view: 'v1' }]); + await rpc('get_state'); + assert.strictEqual(calls.filter(c => c[0] === 'result_open').length, 2); + }); +}); + +suite('Data Explorer translation', () => { + test('every row filter type maps to a kernel filter', () => { + const cases: Array<[RowFilter, unknown]> = [ + [filter('not_between', 0, { left_value: '1', right_value: '2' }), { type: 'between', column: 0, low: '1', high: '2', negate: true }], + [filter('is_null', 1), { type: 'is_null', column: 1, negate: false }], + [filter('not_null', 1), { type: 'is_null', column: 1, negate: true }], + [filter('is_empty', 2), { type: 'is_empty', column: 2, negate: false }], + [filter('not_empty', 2), { type: 'is_empty', column: 2, negate: true }], + [filter('is_true', 0), { type: 'is_true', column: 0 }], + [filter('is_false', 0), { type: 'is_false', column: 0 }], + [filter('search', 2, { search_type: 'regex_match', term: '^a', case_sensitive: true }), { type: 'text', column: 2, match: 'regex', term: '^a', case_sensitive: true }], + [filter('search', 2, { search_type: 'not_contains', term: 'b', case_sensitive: false }), { type: 'text', column: 2, match: 'not_contains', term: 'b', case_sensitive: false }], + [filter('set_membership', 2, { values: ['a'], inclusive: true }), { type: 'in', column: 2, values: ['a'], negate: false }], + [filter('set_membership', 2, { values: ['a'], inclusive: false }), { type: 'in', column: 2, values: ['a'], negate: true }], + [filter('compare', 0, { op: '~', value: '1' }), { error: 'Unknown comparison "~"' }], + [filter('compare', 0, { op: '=' }), { error: 'The filter is missing a value' }], + ]; + for (const [input, expected] of cases) { + assert.deepStrictEqual(toKernelFilter(input), expected, input.filter_type); + } + }); + + test('search_schema matches names and types, and sorts', () => { + const search = (filters: object[], sort_order = 'original') => + searchSchema({ filters, sort_order } as never, COLUMNS); + const text = (search_type: string, term: string, case_sensitive = false) => + ({ filter_type: 'text_search', params: { search_type, term, case_sensitive } }); + + assert.deepStrictEqual(search([]), [0, 1, 2]); + assert.deepStrictEqual(search([text('contains', 'l')]), [2]); + assert.deepStrictEqual(search([text('contains', 'B', true)]), []); + assert.deepStrictEqual(search([text('not_contains', 'n')]), [1, 2]); + assert.deepStrictEqual(search([text('starts_with', 'la')]), [2]); + assert.deepStrictEqual(search([text('ends_with', 'X')]), [1]); + assert.deepStrictEqual(search([text('regex_match', '^[nx]$')]), [0, 1]); + assert.deepStrictEqual(search([text('regex_match', '(')]), []); + assert.deepStrictEqual( + search([{ filter_type: 'match_data_types', params: { display_types: ['string', 'floating'] } }]), + [1, 2] + ); + assert.deepStrictEqual(search([], 'descending_name'), [1, 0, 2]); + assert.deepStrictEqual(search([], 'ascending_type'), [1, 0, 2]); + }); + + test('large profiles get a kernel call of their own', () => { + const calls = toProfileCalls([ + { profile_type: 'small_histogram', params: { method: 'fixed', num_bins: 10 } }, + { profile_type: 'large_histogram', params: { method: 'fixed', num_bins: 100 } }, + { profile_type: 'large_frequency_table', params: { limit: 50 } }, + ]); + assert.deepStrictEqual(calls, [ + { request: { histogram: { bins: 10, quantiles: [] } }, histogramKey: 'small_histogram' }, + { + request: { histogram: { bins: 100, quantiles: [] }, frequency: { limit: 50 } }, + histogramKey: 'large_histogram', + frequencyKey: 'large_frequency_table', + }, + ]); + }); + + test('summaries take the shape of their column kind', () => { + assert.deepStrictEqual(toSummaryStats('boolean', { true_count: 3, false_count: 1 }), { + type_display: 'boolean', boolean_stats: { true_count: 3, false_count: 1 }, + }); + assert.deepStrictEqual(toSummaryStats('string', { num_unique: 3, num_empty: 0 }), { + type_display: 'string', string_stats: { num_unique: 3, num_empty: 0 }, + }); + assert.deepStrictEqual(toSummaryStats('date', { min: '2024-01-01', max: '2024-02-01', num_unique: 2 }), { + type_display: 'date', date_stats: { min_date: '2024-01-01', max_date: '2024-02-01', num_unique: 2 }, + }); + assert.deepStrictEqual(toSummaryStats('datetime', {}).datetime_stats, { + min_date: undefined, max_date: undefined, num_unique: undefined, + }); + assert.deepStrictEqual(toSummaryStats('time', {}), { type_display: 'time' }); + }); + + test('cells map nulls and non-finite floats to their codes', () => { + assert.strictEqual(cellValue(null, 'string'), 0); + assert.strictEqual(cellValue('NaN', 'floating'), 2); + assert.strictEqual(cellValue('inf', 'floating'), 10); + assert.strictEqual(cellValue('-inf', 'floating'), 11); + assert.strictEqual(cellValue('-inf', 'string'), '-inf'); + assert.strictEqual(cellValue('2.5', 'floating'), '2.5'); + }); +}); diff --git a/ggsql-vscode/src/test/manager.test.ts b/ggsql-vscode/src/test/manager.test.ts index 812b0daaf..4caff9db0 100644 --- a/ggsql-vscode/src/test/manager.test.ts +++ b/ggsql-vscode/src/test/manager.test.ts @@ -1,6 +1,6 @@ import * as assert from 'assert'; import * as vscode from 'vscode'; -import { GgsqlRuntimeManager, createDynState, getSupervisorApi } from '../manager'; +import { GgsqlRuntimeManager, consoleRowsEnv, createDynState, getSupervisorApi } from '../manager'; suite('manager', () => { // The suites run in stock VS Code, where positron.positron-supervisor is @@ -13,6 +13,17 @@ suite('manager', () => { ); }); + test('consoleRowsEnv passes a row count the kernel accepts', () => { + assert.deepStrictEqual(consoleRowsEnv(25), { GGSQL_CONSOLE_ROWS: '25' }); + }); + + test('consoleRowsEnv leaves out a value the kernel would refuse', () => { + // The kernel exits on a bad value, so the session would fail to start. + for (const rows of [0, -1, 2.5, '10', undefined, null]) { + assert.deepStrictEqual(consoleRowsEnv(rows), {}, String(rows)); + } + }); + test('createDynState falls back to the default session name', () => { const state = createDynState(); assert.strictEqual(state.sessionName, 'ggsql'); diff --git a/ggsql-vscode/src/test/resultLinks.test.ts b/ggsql-vscode/src/test/resultLinks.test.ts new file mode 100644 index 000000000..8e1fdd77a --- /dev/null +++ b/ggsql-vscode/src/test/resultLinks.test.ts @@ -0,0 +1,132 @@ +import * as assert from 'assert'; +import * as vscode from 'vscode'; +import type * as positron from '@posit-dev/positron'; +import { parseResultLink, resultKey, withResultLinks } from '../resultLinks'; +import { ResultExplorer } from '../dataExplorer/handler'; + +suite('parseResultLink', () => { + test('reads the session id and result key of a link', () => { + assert.deepStrictEqual( + parseResultLink('x-ggsql-result:ggsql-1a2b/0f9e8d'), + { sessionId: 'ggsql-1a2b', key: '0f9e8d' } + ); + }); + + test('ignores other schemes, the old one included', () => { + assert.strictEqual(parseResultLink('x-r-help:topic'), undefined); + assert.strictEqual(parseResultLink('https://ggsql.org'), undefined); + assert.strictEqual(parseResultLink('x-ggsql-view:s/abc'), undefined); + }); + + test('rejects a key the kernel could not have produced', () => { + assert.strictEqual(parseResultLink('x-ggsql-result:s/abc%0ASELECT%201'), undefined); + assert.strictEqual(parseResultLink('x-ggsql-result:s/ABC'), undefined); + assert.strictEqual(parseResultLink('x-ggsql-result:s'), undefined); + assert.strictEqual(parseResultLink('x-ggsql-result:s/abc/def'), undefined); + }); +}); + +type Message = positron.LanguageRuntimeMessage & Record; + +function message(id: string, type: string, extra: Record = {}): Message { + return { + id, + parent_id: 'exec-1', + when: '2026-01-01T00:00:00Z', + type: type as positron.LanguageRuntimeMessageType, + ...extra, + }; +} + +function result(id: string, key?: unknown): Message { + return message(id, 'result', { + data: { 'text/plain': 'table' }, + execution_count: 1, + outputMetadata: key === undefined ? {} : { ggsql: { result: key } }, + }); +} + +/** A stand-in supervisor session, with the received messages under `fire`. */ +function fakeSession(sessionMode: string) { + const messages = new vscode.EventEmitter(); + const ended = new vscode.EventEmitter(); + let disposed = false; + const session = { + metadata: { sessionId: 'ggsql-abc', sessionMode }, + onDidReceiveRuntimeMessage: messages.event, + onDidEndSession: ended.event, + callMethod: async () => ({}), + dispose() { + disposed = true; + }, + whoAmI() { + return this; + }, + }; + return { + session: session as unknown as positron.LanguageRuntimeSession, + raw: session, + fire: (m: positron.LanguageRuntimeMessage) => messages.fire(m), + disposed: () => disposed, + }; +} + +function received(sessionMode: string, input: Message[]): Message[] { + const fake = fakeSession(sessionMode); + const wrapped = withResultLinks(fake.session, new ResultExplorer(() => { })); + const out: Message[] = []; + wrapped.onDidReceiveRuntimeMessage(m => out.push(m as Message)); + input.forEach(fake.fire); + wrapped.dispose(); + return out; +} + +suite('withResultLinks', () => { + test('follows a marked result with a link to it, in a console', () => { + const r = result('r1', '0f9e'); + const out = received('console', [message('i1', 'input'), r, message('s1', 'state')]); + assert.deepStrictEqual(out.map(m => m.id), ['i1', 'r1', 'r1-ggsql-result-link', 's1']); + assert.strictEqual(out[1], r); + const link = out[2]; + assert.strictEqual(link.type, 'stream'); + assert.strictEqual(link.name, 'stdout'); + assert.strictEqual(link.parent_id, 'exec-1'); + assert.strictEqual(link.when, r.when); + assert.strictEqual( + link.text, + '\x1b]8;;x-ggsql-result:ggsql-abc/0f9e\x1b\\Open in Data Explorer\x1b]8;;\x1b\\\n' + ); + }); + + test('adds nothing outside a console', () => { + for (const mode of ['notebook', 'background']) { + const out = received(mode, [result('r1', '0f9e')]); + assert.deepStrictEqual(out.map(m => m.id), ['r1'], mode); + } + }); + + test('adds nothing for a result without a valid key', () => { + const out = received('console', [ + result('r1'), + result('r2', 'NOT HEX'), + result('r3', 42), + message('o1', 'output', { outputMetadata: { ggsql: { result: '0f9e' } } }), + ]); + assert.deepStrictEqual(out.map(m => m.id), ['r1', 'r2', 'r3', 'o1']); + }); + + test('passes everything else through, bound to the session', async () => { + const fake = fakeSession('console'); + const wrapped = withResultLinks(fake.session, new ResultExplorer(() => { })); + assert.strictEqual(wrapped.metadata.sessionId, 'ggsql-abc'); + assert.strictEqual((wrapped as unknown as typeof fake.raw).whoAmI(), fake.raw); + assert.strictEqual(await wrapped.openResource!('https://ggsql.org'), false); + wrapped.dispose(); + assert.ok(fake.disposed()); + }); + + test('resultKey reads only result messages', () => { + assert.strictEqual(resultKey(result('r', 'ab12')), 'ab12'); + assert.strictEqual(resultKey(message('o', 'output', { outputMetadata: { ggsql: { result: 'ab' } } })), undefined); + }); +}); diff --git a/src/AGENTS.md b/src/AGENTS.md index 746e29e9e..d99f9d860 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -59,7 +59,7 @@ Grammar lives in [`/tree-sitter-ggsql/`](../tree-sitter-ggsql/) — when adding `SqlDialect` trait in `mod.rs` lets each driver supply its own type names, information-schema queries, and spatial helper methods (`sql_st_transform`, `sql_geometry_to_wkb`, `sql_geometry_bbox`, `sql_ensure_geometry`, `sql_select_replace`, `sql_spatial_setup`). -**Caching layer.** `CachingReader` (`cache.rs`) wraps a primary reader plus an in-memory `CacheBackend`, splitting work across two `Reader` surfaces. **`execute_sql` = source**: base reads of the user's data plus user setup/DML run on the primary (with result memoization), except `ggsql:` builtins, the `__ggsql_cache_meta__` table, and reads that reference a cache-resident internal table, which go to the cache. **`execute_sql_cached` = compute**: all dialect-generated/derived SQL (schema probes, stats, projection/map transforms, spatial setup, final layer queries — everything operating on `__ggsql_*` tables) runs on the cache; it defaults to `execute_sql` so a plain reader runs everything on one connection. Cache routing is by **exact-identifier membership** in the set of tables registered into the cache. Memoization keys on `hash(primary_uri + sql)` and is tracked in the `__ggsql_cache_meta__` table inside the cache backend. Each memoized read is bounded by a **TTL** (default 300s) and the whole memo by an **LRU byte budget** (default 512 MB); both are configurable via `CacheConfig` (env `GGSQL_CACHE_DISABLED`/`GGSQL_CACHE_TTL`/`GGSQL_CACHE_MAX_BYTES`, or per-connection URI query parameters `?cache_ttl=…&cache_max_bytes=…&cache_disabled=…`). The `__ggsql_cache_meta__` table is queryable for introspection (`SELECT * FROM __ggsql_cache_meta__`). Pure/non-visual SQL (CLI table fallback, Jupyter) goes through `execute_sql` so it reads the primary rather than the empty cache. `Reader::materialize_table` (default = `CREATE TEMP TABLE` on the reader, no Rust roundtrip) is overridden to read the body via the source surface and `register()` the result into the cache, so the primary is never written to; `Reader::caches_sources()` (default `false`, `true` for `CachingReader`) gates the executor's per-layer source staging: file sources are staged on the cache surface, while identifiers go through `materialize_table`, which routes the read to the cache (CTEs, builtins, cache-resident tables) or the primary as needed. `dialect()` returns the **cache** dialect, and every compute-surface failure is prefixed with the cache backend's scheme (``on the `duckdb` cache backend: …``) so a cache-dialect driver error is not mistaken for one from the user's own connection. Selected via the composite `+://` scheme (`reader_from_uri` / `split_cache_uri`) or the CLI `--cache` flag; off by default. +**Caching layer.** `CachingReader` (`cache.rs`) wraps a primary reader plus an in-memory `CacheBackend`, splitting work across two `Reader` surfaces. **`execute_sql` = source**: base reads of the user's data plus user setup/DML run on the primary (with result memoization), except `ggsql:` builtins, the `__ggsql_cache_meta__` table, and reads that reference a cache-resident internal table, which go to the cache. **`execute_sql_cached` = compute**: all dialect-generated/derived SQL (schema probes, stats, projection/map transforms, spatial setup, final layer queries — everything operating on `__ggsql_*` tables) runs on the cache; it defaults to `execute_sql` so a plain reader runs everything on one connection. Cache routing is by **exact-identifier membership** in the set of tables registered into the cache. Memoization keys on `hash(primary_uri + sql)` and is tracked in the `__ggsql_cache_meta__` table inside the cache backend. Each memoized read is bounded by a **TTL** (default 300s) and the whole memo by an **LRU byte budget** (default 512 MB); both are configurable via `CacheConfig` (env `GGSQL_CACHE_DISABLED`/`GGSQL_CACHE_TTL`/`GGSQL_CACHE_MAX_BYTES`, or per-connection URI query parameters `?cache_ttl=…&cache_max_bytes=…&cache_disabled=…`). The `__ggsql_cache_meta__` table is queryable for introspection (`SELECT * FROM __ggsql_cache_meta__`). Pure/non-visual SQL (CLI table fallback, Jupyter) goes through `execute_sql` so it reads the primary rather than the empty cache. `Reader::materialize_table` (default = `CREATE TEMP TABLE` on the reader, no Rust roundtrip) is overridden to read the body via the source surface and `register()` the result into the cache, so the primary is never written to; `Reader::caches_sources()` (default `false`, `true` for `CachingReader`) gates the executor's per-layer source staging: file sources are staged on the cache surface, while identifiers go through `materialize_table`, which routes the read to the cache (CTEs, builtins, cache-resident tables) or the primary as needed. `dialect()` returns the **cache** dialect, and every compute-surface failure is prefixed with the cache backend's scheme (``on the `duckdb` cache backend: …``) so a cache-dialect driver error is not mistaken for one from the user's own connection. Selected via the composite `+://` scheme (`reader_from_uri` / `split_cache_uri`) or the CLI `--cache` flag; off by default. The memo's bookkeeping — registering a table, its row in `__ggsql_cache_meta__`, LRU eviction under the byte budget — is `CacheTables`, which other callers use on a backend of their own (the Jupyter kernel's results API does, without wrapping the user's reader). An entry can be **pinned**, exempting it from eviction until unpinned; `CachingReader` never pins, and the TTL check stays in `CachingReader`. ### `execute/` diff --git a/src/reader/cache.rs b/src/reader/cache.rs index f38164a57..1eda4fc38 100644 --- a/src/reader/cache.rs +++ b/src/reader/cache.rs @@ -127,112 +127,88 @@ fn estimate_bytes(df: &DataFrame) -> i64 { } /// One memo row, trimmed to the fields consulted at runtime. -struct MemoEntry { - table_name: String, - fetched_at_epoch_ms: i64, +pub struct MemoEntry { + pub table_name: String, + pub fetched_at_epoch_ms: i64, } -pub struct CachingReader { - /// Primary backend — the real data source. - primary: Box, - /// In-memory writeable cache: derived tables, registered data, memoized reads. - cache: Box, - /// Connection URI of the primary. - primary_uri: String, - /// Scheme of the cache backend as spelled in the connection URI. - cache_scheme: String, - /// TTL + byte-budget configuration for the result memo. +/// Tables kept in an in-memory cache backend under a byte budget. +/// +/// Every entry is a table in the backend plus a row in the +/// `__ggsql_cache_meta__` table, which records its size and when it was last +/// used. Once the entries' total size passes the budget the least recently +/// used are dropped, except entries that are **pinned**: those stay until +/// unpinned, even past the budget. +/// +/// [`CachingReader`] keeps memoized reads here, keyed by a hash of their SQL, +/// and never pins. Other callers can keep tables in a `CacheTables` of their +/// own without wrapping any reader, so nothing changes about the dialect a +/// user's queries run in. +pub struct CacheTables { + /// The in-memory backend holding the tables and the metadata table. + backend: Box, + /// TTL + byte-budget configuration. Checking the TTL is the caller's job. config: CacheConfig, - /// Whether the metadata table has been created in the cache backend. + /// Whether the metadata table has been created in the backend. meta_ready: Cell, - /// Names registered into the cache. A source read that references one - /// is routed to the cache rather than the primary. - resident: RefCell>, } -impl CachingReader { - /// Construct a `CachingReader` from a primary reader, an in-memory cache - /// backend, the primary's connection URI and the cache backend's scheme, - /// using environment-derived cache configuration. The cache is owned by the - /// `CachingReader` and dropped with it. - /// - /// Test-only convenience; production construction goes through - /// [`connection`](crate::reader::connection), which always supplies an - /// explicit [`CacheConfig`] via [`with_config`](Self::with_config). - #[cfg(test)] - pub fn new( - primary: Box, - cache: Box, - primary_uri: impl Into, - cache_scheme: impl Into, - ) -> Self { - Self::with_config( - primary, - cache, - primary_uri, - cache_scheme, - CacheConfig::from_env(), - ) - } - - /// Construct a `CachingReader` with explicit cache configuration. - pub fn with_config( - primary: Box, - cache: Box, - primary_uri: impl Into, - cache_scheme: impl Into, - config: CacheConfig, - ) -> Self { +impl CacheTables { + pub fn new(backend: Box, config: CacheConfig) -> Self { Self { - primary, - cache, - primary_uri: primary_uri.into(), - cache_scheme: cache_scheme.into(), + backend, config, meta_ready: Cell::new(false), - resident: RefCell::new(HashSet::new()), } } - /// The active cache configuration. - pub fn cache_config(&self) -> &CacheConfig { - &self.config + /// A `CacheTables` on a fresh in-memory backend: DuckDB when the `duckdb` + /// feature is compiled in, SQLite otherwise. + #[cfg(any(feature = "duckdb", feature = "sqlite"))] + pub fn in_memory(config: CacheConfig) -> Result { + use crate::reader::connection::{build_reader, cache_uri, default_cache_scheme}; + let backend = build_reader(cache_uri(default_cache_scheme())?)?; + Ok(Self::new(backend, config)) } - /// Derive a stable cache key from the primary URI and the SQL text. - fn cache_key(&self, sql: &str) -> String { - let mut hasher = DefaultHasher::new(); - hasher.write(self.primary_uri.as_bytes()); - hasher.write(b"\n"); - hasher.write(sql.as_bytes()); - format!("{:016x}", hasher.finish()) + /// The backend holding the tables, for queries over them. + pub fn backend(&self) -> &dyn Reader { + &*self.backend } - /// Create the metadata table in the cache backend if it doesn't exist yet. - fn ensure_meta_table(&self) -> Result<()> { + /// The active configuration. + pub fn config(&self) -> &CacheConfig { + &self.config + } + + /// Create the metadata table in the backend if it doesn't exist yet. + pub fn ensure_meta_table(&self) -> Result<()> { if self.meta_ready.get() { return Ok(()); } + // `pinned` is an integer rather than a BOOLEAN so the same DDL and + // comparisons work on every cache backend. let sql = format!( "CREATE TABLE IF NOT EXISTS {} (\ cache_key VARCHAR PRIMARY KEY, sql VARCHAR NOT NULL, table_name VARCHAR NOT NULL, \ fetched_at_epoch_ms BIGINT NOT NULL, last_accessed_epoch_ms BIGINT NOT NULL, \ - byte_estimate BIGINT NOT NULL, row_count BIGINT NOT NULL)", + byte_estimate BIGINT NOT NULL, row_count BIGINT NOT NULL, pinned BIGINT NOT NULL)", naming::quote_ident(naming::CACHE_META_TABLE) ); - self.cache.execute_sql(&sql)?; + self.backend.execute_sql(&sql)?; self.meta_ready.set(true); Ok(()) } - /// Look up the memo entry for `key`. - fn lookup_memo(&self, key: &str) -> Result> { + /// Look up the entry for `key`. + pub fn lookup(&self, key: &str) -> Result> { + self.ensure_meta_table()?; let sql = format!( "SELECT table_name, fetched_at_epoch_ms FROM {} WHERE cache_key = {}", naming::quote_ident(naming::CACHE_META_TABLE), naming::quote_literal(key), ); - let df = self.cache.execute_sql(&sql)?; + let df = self.backend.execute_sql(&sql)?; if df.height() == 0 { return Ok(None); } @@ -244,14 +220,39 @@ impl CachingReader { })) } - /// Record a memoized read in the metadata table. - fn insert_memo( + /// Keep `df` as the table `table` under `key`, labelled with the `sql` + /// that produced it, then evict over the budget. A `pinned` entry is never + /// evicted, so it survives the eviction its own insertion triggers. + pub fn put( + &self, + key: &str, + sql: &str, + table: &str, + df: DataFrame, + pinned: bool, + ) -> Result<()> { + self.ensure_meta_table()?; + let byte_estimate = estimate_bytes(&df); + let row_count = df.height() as i64; + self.backend.register(table, df, true)?; + if let Err(e) = self.insert_meta(key, sql, table, byte_estimate, row_count, pinned) { + // The table registered but the meta row didn't: drop the orphan so + // it can't leak, then surface the error. + let _ = self.backend.unregister(table); + return Err(e); + } + self.evict_over_budget() + } + + /// Record an entry in the metadata table. + fn insert_meta( &self, key: &str, sql: &str, table: &str, byte_estimate: i64, row_count: i64, + pinned: bool, ) -> Result<()> { let now = now_ms(); // Upsert as DELETE + INSERT rather than INSERT OR REPLACE: the cache @@ -263,12 +264,12 @@ impl CachingReader { naming::quote_ident(naming::CACHE_META_TABLE), naming::quote_literal(key), ); - self.cache.execute_sql(&del)?; + self.backend.execute_sql(&del)?; let stmt = format!( "INSERT INTO {} \ (cache_key, sql, table_name, fetched_at_epoch_ms, last_accessed_epoch_ms, \ - byte_estimate, row_count) \ - VALUES ({}, {}, {}, {}, {}, {}, {})", + byte_estimate, row_count, pinned) \ + VALUES ({}, {}, {}, {}, {}, {}, {}, {})", naming::quote_ident(naming::CACHE_META_TABLE), naming::quote_literal(key), naming::quote_literal(sql), @@ -277,44 +278,65 @@ impl CachingReader { now, byte_estimate, row_count, + i64::from(pinned), ); - self.cache.execute_sql(&stmt)?; + self.backend.execute_sql(&stmt)?; Ok(()) } /// Advance the last-accessed timestamp for `key` (LRU bookkeeping). - fn touch(&self, key: &str) -> Result<()> { + pub fn touch(&self, key: &str) -> Result<()> { let stmt = format!( "UPDATE {} SET last_accessed_epoch_ms = {} WHERE cache_key = {}", naming::quote_ident(naming::CACHE_META_TABLE), now_ms(), naming::quote_literal(key), ); - self.cache.execute_sql(&stmt)?; + self.backend.execute_sql(&stmt)?; + Ok(()) + } + + /// Pin or unpin the entry for `key`, which also counts as a use of it. + /// Pinned entries are never evicted. Unpinning evicts over the budget, + /// since the entry counts towards it again. + pub fn set_pinned(&self, key: &str, pinned: bool) -> Result<()> { + self.ensure_meta_table()?; + let stmt = format!( + "UPDATE {} SET pinned = {}, last_accessed_epoch_ms = {} WHERE cache_key = {}", + naming::quote_ident(naming::CACHE_META_TABLE), + i64::from(pinned), + now_ms(), + naming::quote_literal(key), + ); + self.backend.execute_sql(&stmt)?; + if !pinned { + self.evict_over_budget()?; + } Ok(()) } - /// Drop a single memo entry: unregister the table, then delete its meta row. - fn drop_entry(&self, key: &str, table: &str) -> Result<()> { - self.cache.unregister(table)?; + /// Drop a single entry: unregister the table, then delete its meta row. + pub fn drop_entry(&self, key: &str, table: &str) -> Result<()> { + self.backend.unregister(table)?; let del = format!( "DELETE FROM {} WHERE cache_key = {}", naming::quote_ident(naming::CACHE_META_TABLE), naming::quote_literal(key), ); - self.cache.execute_sql(&del)?; + self.backend.execute_sql(&del)?; Ok(()) } - /// Evict LRU entries until the cumulative byte estimate is within `max_bytes`. - fn evict_over_budget(&self) -> Result<()> { + /// Evict unpinned LRU entries until the cumulative byte estimate is within + /// `max_bytes`, or nothing unpinned is left. + pub fn evict_over_budget(&self) -> Result<()> { // Cast the SUM to BIGINT. let sum_sql = format!( "SELECT CAST(COALESCE(SUM(byte_estimate), 0) AS BIGINT) AS n FROM {}", naming::quote_ident(naming::CACHE_META_TABLE) ); loop { - let df = self.cache.execute_sql(&sum_sql)?; + let df = self.backend.execute_sql(&sum_sql)?; let total = if df.height() == 0 { 0 } else { @@ -324,10 +346,11 @@ impl CachingReader { return Ok(()); } let pick = format!( - "SELECT cache_key, table_name FROM {} ORDER BY last_accessed_epoch_ms ASC LIMIT 1", + "SELECT cache_key, table_name FROM {} WHERE pinned = 0 \ + ORDER BY last_accessed_epoch_ms ASC LIMIT 1", naming::quote_ident(naming::CACHE_META_TABLE) ); - let df = self.cache.execute_sql(&pick)?; + let df = self.backend.execute_sql(&pick)?; if df.height() == 0 { return Ok(()); } @@ -337,30 +360,10 @@ impl CachingReader { } } - /// Whether `sql` reads something the cache backend owns: a cache-resident - /// table, the metadata table, or a `ggsql:` builtin dataset. - /// - /// Determined from the parsed `table_ref`/namespaced-identifier targets. - /// On a parse failure we conservatively return `false`. - fn references_cache_resident(&self, sql: &str) -> bool { - if crate::parser::extract_builtin_dataset_names(sql) - .map(|d| !d.is_empty()) - .unwrap_or(false) - { - return true; - } - let Ok(refs) = crate::parser::extract_table_refs(sql) else { - return false; - }; - let resident = self.resident.borrow(); - refs.iter() - .any(|t| t == naming::CACHE_META_TABLE || resident.contains(t)) - } - - /// Drop every memoized result, one entry at a time. - fn clear_memo(&self) -> Result<()> { + /// Drop every entry, one at a time. + pub fn clear(&self) -> Result<()> { self.ensure_meta_table()?; - let df = self.cache.execute_sql(&format!( + let df = self.backend.execute_sql(&format!( "SELECT cache_key, table_name FROM {}", naming::quote_ident(naming::CACHE_META_TABLE) ))?; @@ -387,6 +390,103 @@ impl CachingReader { } Ok(()) } +} + +pub struct CachingReader { + /// Primary backend — the real data source. + primary: Box, + /// In-memory writeable cache: derived tables, registered data, memoized + /// reads, and the memo's bookkeeping. + tables: CacheTables, + /// Connection URI of the primary. + primary_uri: String, + /// Scheme of the cache backend as spelled in the connection URI. + cache_scheme: String, + /// Names registered into the cache. A source read that references one + /// is routed to the cache rather than the primary. + resident: RefCell>, +} + +impl CachingReader { + /// Construct a `CachingReader` from a primary reader, an in-memory cache + /// backend, the primary's connection URI and the cache backend's scheme, + /// using environment-derived cache configuration. The cache is owned by the + /// `CachingReader` and dropped with it. + /// + /// Test-only convenience; production construction goes through + /// [`connection`](crate::reader::connection), which always supplies an + /// explicit [`CacheConfig`] via [`with_config`](Self::with_config). + #[cfg(test)] + pub fn new( + primary: Box, + cache: Box, + primary_uri: impl Into, + cache_scheme: impl Into, + ) -> Self { + Self::with_config( + primary, + cache, + primary_uri, + cache_scheme, + CacheConfig::from_env(), + ) + } + + /// Construct a `CachingReader` with explicit cache configuration. + pub fn with_config( + primary: Box, + cache: Box, + primary_uri: impl Into, + cache_scheme: impl Into, + config: CacheConfig, + ) -> Self { + Self { + primary, + tables: CacheTables::new(cache, config), + primary_uri: primary_uri.into(), + cache_scheme: cache_scheme.into(), + resident: RefCell::new(HashSet::new()), + } + } + + /// The active cache configuration. + pub fn cache_config(&self) -> &CacheConfig { + self.tables.config() + } + + /// The cache backend. + fn cache(&self) -> &dyn Reader { + self.tables.backend() + } + + /// Derive a stable cache key from the primary URI and the SQL text. + fn cache_key(&self, sql: &str) -> String { + let mut hasher = DefaultHasher::new(); + hasher.write(self.primary_uri.as_bytes()); + hasher.write(b"\n"); + hasher.write(sql.as_bytes()); + format!("{:016x}", hasher.finish()) + } + + /// Whether `sql` reads something the cache backend owns: a cache-resident + /// table, the metadata table, or a `ggsql:` builtin dataset. + /// + /// Determined from the parsed `table_ref`/namespaced-identifier targets. + /// On a parse failure we conservatively return `false`. + fn references_cache_resident(&self, sql: &str) -> bool { + if crate::parser::extract_builtin_dataset_names(sql) + .map(|d| !d.is_empty()) + .unwrap_or(false) + { + return true; + } + let Ok(refs) = crate::parser::extract_table_refs(sql) else { + return false; + }; + let resident = self.resident.borrow(); + refs.iter() + .any(|t| t == naming::CACHE_META_TABLE || resident.contains(t)) + } /// Prefix `context` onto a backend error, dropping the driver's own /// "Failed to {execute,prepare} SQL" preamble so the attribution reads first. @@ -405,16 +505,16 @@ impl CachingReader { impl Reader for CachingReader { /// Source surface: base reads of the user's data (plus user setup/DML). fn execute_sql(&self, sql: &str) -> Result { - self.ensure_meta_table()?; + self.tables.ensure_meta_table()?; // Route to the cache when the read targets cache-resident objects, the // metadata table, or a builtin dataset. if self.references_cache_resident(sql) { - return self.cache.execute_sql(sql); + return self.cache().execute_sql(sql); } // With caching disabled the memo is never consulted or written. - if !self.config.enabled { + if !self.tables.config().enabled { return self.primary.execute_sql(sql); } @@ -422,17 +522,17 @@ impl Reader for CachingReader { // Serve a fresh, still-present entry; on a stale or vanished entry drop // it and fall through to a primary re-fetch. - if let Some(entry) = self.lookup_memo(&key)? { + if let Some(entry) = self.tables.lookup(&key)? { let age_ms = (now_ms() - entry.fetched_at_epoch_ms).max(0); - let ttl_ms = (self.config.ttl_secs as i64).saturating_mul(1000); + let ttl_ms = (self.tables.config().ttl_secs as i64).saturating_mul(1000); if age_ms < ttl_ms { let select = format!("SELECT * FROM {}", naming::quote_ident(&entry.table_name)); - if let Ok(df) = self.cache.execute_sql(&select) { - self.touch(&key)?; + if let Ok(df) = self.cache().execute_sql(&select) { + self.tables.touch(&key)?; return Ok(df); } } - self.drop_entry(&key, &entry.table_name)?; + self.tables.drop_entry(&key, &entry.table_name)?; } let df = self.primary.execute_sql(sql)?; @@ -440,16 +540,7 @@ impl Reader for CachingReader { // Cache row-returning reads only. if super::returns_rows(sql) && df.width() > 0 { let table = naming::cache_result_table(&key); - let byte_estimate = estimate_bytes(&df); - let row_count = df.height() as i64; - self.cache.register(&table, df.clone(), true)?; - if let Err(e) = self.insert_memo(&key, sql, &table, byte_estimate, row_count) { - // The result table registered but the meta row didn't: drop the - // orphan so it can't leak, then surface the error. - let _ = self.cache.unregister(&table); - return Err(e); - } - self.evict_over_budget()?; + self.tables.put(&key, sql, &table, df.clone(), false)?; } Ok(df) @@ -457,19 +548,19 @@ impl Reader for CachingReader { /// Compute surface: derived/dialect-generated SQL runs on the cache. fn execute_sql_cached(&self, sql: &str) -> Result { - self.cache.execute_sql(sql).map_err(|e| { + self.cache().execute_sql(sql).map_err(|e| { self.annotate_error(&format!("on the `{}` cache backend", self.cache_scheme), e) }) } fn register(&self, name: &str, df: DataFrame, replace: bool) -> Result<()> { - self.cache.register(name, df, replace)?; + self.cache().register(name, df, replace)?; self.resident.borrow_mut().insert(name.to_string()); Ok(()) } fn unregister(&self, name: &str) -> Result<()> { - self.cache.unregister(name)?; + self.cache().unregister(name)?; self.resident.borrow_mut().remove(name); Ok(()) } @@ -480,7 +571,7 @@ impl Reader for CachingReader { fn dialect(&self) -> &dyn SqlDialect { // All executor-generated SQL targets cache-resident tables. - self.cache.dialect() + self.cache().dialect() } fn materialize_table( @@ -507,7 +598,7 @@ impl Reader for CachingReader { } fn clear_cache(&self) -> Result<()> { - self.clear_memo() + self.tables.clear() } // Schema introspection describes the real data source, so delegate to the @@ -1310,7 +1401,7 @@ mod behavior_tests { .value(0) .to_string(); reader - .cache + .cache() .execute_sql(&format!("DROP TABLE {}", naming::quote_ident(&table_name))) .unwrap(); @@ -1326,6 +1417,49 @@ mod behavior_tests { assert_eq!(hits, 2, "the missing table forced a primary re-fetch"); } + fn tiny_budget_tables() -> CacheTables { + CacheTables::new( + Box::new(DuckDBReader::new_in_memory().unwrap()), + CacheConfig { + enabled: true, + ttl_secs: 300, + max_bytes: 1, + }, + ) + } + + #[test] + fn test_pinned_entries_survive_eviction() { + let tables = tiny_budget_tables(); + let frame = || df! { "x" => vec![1_i64, 2, 3] }.unwrap(); + tables.put("a", "a", "__t_a__", frame(), true).unwrap(); + tables.put("b", "b", "__t_b__", frame(), false).unwrap(); + tables.put("c", "c", "__t_c__", frame(), true).unwrap(); + + // Over a 1-byte budget, only the unpinned entry goes. + assert!(tables.lookup("a").unwrap().is_some()); + assert!(tables.lookup("b").unwrap().is_none()); + assert!(tables.lookup("c").unwrap().is_some()); + let rows = tables + .backend() + .execute_sql("SELECT x FROM __t_a__") + .unwrap(); + assert_eq!(rows.height(), 3); + } + + #[test] + fn test_unpinning_makes_an_entry_evictable() { + let tables = tiny_budget_tables(); + let frame = || df! { "x" => vec![1_i64] }.unwrap(); + tables.put("a", "a", "__t_a__", frame(), true).unwrap(); + tables.set_pinned("a", false).unwrap(); + assert!(tables.lookup("a").unwrap().is_none()); + assert!(tables + .backend() + .execute_sql("SELECT x FROM __t_a__") + .is_err()); + } + #[test] fn test_parse_human_bytes() { assert_eq!(parse_human_bytes("1024"), Some(1024)); diff --git a/src/reader/connection.rs b/src/reader/connection.rs index 17e69be66..79bbca4ce 100644 --- a/src/reader/connection.rs +++ b/src/reader/connection.rs @@ -197,7 +197,7 @@ impl ConnUri { } /// Map a cache-backend scheme to its in-memory connection URI. -fn cache_uri(scheme: &str) -> Result<&'static str> { +pub(crate) fn cache_uri(scheme: &str) -> Result<&'static str> { match scheme { "duckdb" => Ok("duckdb://memory"), "sqlite" => Ok("sqlite://:memory:"), @@ -209,6 +209,17 @@ fn cache_uri(scheme: &str) -> Result<&'static str> { } } +/// The in-memory backend a cache uses when none is named: DuckDB when the +/// `duckdb` feature is compiled in, SQLite otherwise. +#[cfg(any(feature = "duckdb", feature = "sqlite"))] +pub(crate) fn default_cache_scheme() -> &'static str { + if cfg!(feature = "duckdb") { + "duckdb" + } else { + "sqlite" + } +} + /// Build a reader from a non-composite connection URI pub fn build_reader(uri: &str) -> Result> { let conn = ConnUri::parse(uri)?; @@ -592,11 +603,7 @@ fn auto_cache_if_needed( { use crate::reader::cache::CacheConfig; - let cache_scheme = if cfg!(feature = "duckdb") { - "duckdb" - } else { - "sqlite" - }; + let cache_scheme = default_cache_scheme(); let cache = build_reader(cache_uri(cache_scheme)?)?; Ok(Box::new(crate::reader::CachingReader::with_config( reader, diff --git a/src/reader/duckdb.rs b/src/reader/duckdb.rs index 2e431120f..7567fa307 100644 --- a/src/reader/duckdb.rs +++ b/src/reader/duckdb.rs @@ -162,6 +162,24 @@ impl DuckDBReader { }) } + /// Append `df`'s rows to the temporary table `name` with DuckDB's Arrow + /// appender. + fn append_rows(&self, name: &str, df: &DataFrame) -> Result<()> { + let fail = |e: duckdb::Error| { + GgsqlError::ReaderError(format!("Failed to append to table '{}': {}", name, e)) + }; + // `register` creates temporary tables, which DuckDB keeps in its + // `temp` catalog. + let mut appender = self + .conn + .appender_to_catalog_and_db(name, "temp", "main") + .map_err(fail)?; + appender + .append_record_batch(df.inner().clone()) + .map_err(fail)?; + appender.flush().map_err(fail) + } + /// Get a reference to the underlying DuckDB connection /// /// Useful for executing setup queries (CREATE TABLE, INSERT, etc.) @@ -284,6 +302,30 @@ impl Reader for DuckDBReader { GgsqlError::ReaderError(format!("Failed to register table '{}': {}", name, e)) })?; + // The rest goes through DuckDB's Arrow appender, which is several + // times faster than one `INSERT` per chunk on large frames (it + // chunks to the vector size itself). A type the appender cannot + // convert falls back to the `INSERT` loop, starting over so a + // partial append never leaves a half-filled table. + if self + .append_rows( + name, + &df.slice(MAX_ARROW_BATCH_ROWS, total_rows - MAX_ARROW_BATCH_ROWS), + ) + .is_ok() + { + self.registered_tables.note_registered(name); + return Ok(()); + } + let params = dataframe_to_arrow_params(&first_chunk)?; + let recreate_sql = format!( + "CREATE OR REPLACE TEMP TABLE {} AS SELECT * FROM arrow(?, ?)", + naming::quote_ident(name) + ); + self.conn.execute(&recreate_sql, params).map_err(|e| { + GgsqlError::ReaderError(format!("Failed to register table '{}': {}", name, e)) + })?; + let mut offset = MAX_ARROW_BATCH_ROWS; while offset < total_rows { let chunk_size = std::cmp::min(MAX_ARROW_BATCH_ROWS, total_rows - offset); @@ -452,6 +494,30 @@ mod tests { ); } + #[test] + fn test_register_large_frame_keeps_every_row_in_order() { + // Past one Arrow chunk, the rest goes through the appender. + let reader = DuckDBReader::from_connection_string("duckdb://memory").unwrap(); + let n = 5000; + let df = df! { + "i" => (0..n as i64).collect::>(), + "s" => (0..n).map(|i| format!("row {i}")).collect::>(), + "f" => (0..n).map(|i| (i % 7 == 0).then_some(i as f64 / 2.0)).collect::>(), + } + .unwrap(); + reader.register("big", df, false).unwrap(); + + let out = reader + .execute_sql("SELECT i, s, f FROM big ORDER BY rowid") + .unwrap(); + assert_eq!(out.height(), n); + let i = as_i64(out.column("i").unwrap()).unwrap(); + let s = as_str(out.column("s").unwrap()).unwrap(); + assert!((0..n).all(|r| i.value(r) == r as i64)); + assert_eq!(s.value(4999), "row 4999"); + assert_eq!(out.column("f").unwrap().null_count(), n - n.div_ceil(7)); + } + #[test] fn test_register_duplicate_name_errors() { let reader = DuckDBReader::from_connection_string("duckdb://memory").unwrap(); diff --git a/src/reader/odbc/mod.rs b/src/reader/odbc/mod.rs index 76f420dca..baf11ba13 100644 --- a/src/reader/odbc/mod.rs +++ b/src/reader/odbc/mod.rs @@ -164,7 +164,7 @@ impl Reader for OdbcReader { .fields() .iter() .map(|field| { - let ty = super::register_column_type(&*self.dialect, field.data_type()) + let ty = super::register_column_type(self.dialect, field.data_type()) .unwrap_or_else(|_| "TEXT".to_string()); format!("{} {}", self.dialect.quote_ident(field.name()), ty) }) From 55afac667bedd6c74ce2650a7fe20b76fc056c1e Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Thu, 19 Nov 2026 10:51:40 -0600 Subject: [PATCH 2/8] Address review of the results API - A failed sort or filter leaves its view as it was: the new order table is built under a new name and only then replaces the old one. - Pages are bounded (MAX_PAGE_ROWS), scattered indices fetch just those rows, and an index past the end is an error rather than a shifted column. - One byte budget (--results-max-bytes) covers unopened results, cached ones including pinned, and order tables. Keeping results is opt-in (--keep-results / GGSQL_KEEP_RESULTS), which the extension sets for console sessions. - result_sql quotes for the session dialect, escapes LIKE with '!', keeps a cell's earlier statements, and leaves out repeated column names. Regex support comes from SqlDialect::sql_regex_match. - Views close with the UI comm. The extension releases a view when the Data Explorer fails to open, ignores requests after close, and never calls a session that is down. - CacheTables takes a byte budget, keeps pins in memory, and has a narrower API; the kernel builds its own DuckDB backend for it. - kind_of matches DuckDB type names exactly. - The DuckDB Arrow appender change moves to its own PR. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 +- Cargo.lock | 1 - Cargo.toml | 2 +- doc/get_started/tooling/positron-vscode.qmd | 2 +- ggsql-jupyter/AGENTS.md | 18 +- ggsql-jupyter/Cargo.toml | 3 - ggsql-jupyter/examples/results_bench.rs | 18 +- ggsql-jupyter/src/display.rs | 28 +- ggsql-jupyter/src/kernel.rs | 43 +- ggsql-jupyter/src/main.rs | 43 +- ggsql-jupyter/src/results/filter.rs | 297 +++++++--- ggsql-jupyter/src/results/methods.rs | 82 ++- ggsql-jupyter/src/results/mod.rs | 611 +++++++++++++++----- ggsql-jupyter/src/results/view.rs | 301 ++++++---- ggsql-jupyter/tests/test_integration.py | 91 ++- ggsql-vscode/AGENTS.md | 10 +- ggsql-vscode/package.json | 1 + ggsql-vscode/src/dataExplorer/handler.ts | 65 ++- ggsql-vscode/src/manager.ts | 24 +- ggsql-vscode/src/resultLinks.ts | 45 +- ggsql-vscode/src/test/dataExplorer.test.ts | 31 +- ggsql-vscode/src/test/manager.test.ts | 15 +- ggsql-vscode/src/test/resultLinks.test.ts | 77 ++- src/AGENTS.md | 2 +- src/reader/cache.rs | 194 ++++--- src/reader/connection.rs | 19 +- src/reader/dialects/duckdb.rs | 5 + src/reader/dialects/mod.rs | 20 + src/reader/dialects/postgres.rs | 5 + src/reader/duckdb.rs | 66 --- src/reader/mod.rs | 8 + 31 files changed, 1549 insertions(+), 594 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eae29812..0fda9154f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,12 +25,12 @@ determines whether text boxes get drawn and additional settings `padding`, automatically discover how to write and run ggsql queries (#536). - In a Positron console, a query result can be opened in the Data Explorer from an "Open in Data Explorer" link under its table, with sorting, - filtering, profiles, export and Convert to Code. Long results show their - first 10 rows in the console, configurable with `ggsql.consoleRows`. The - kernel serves this through a frontend-neutral results API, reached with - `call_method` on the UI comm, which keeps opened results in an in-memory - cache of its own and never changes the session's reader or dialect - (posit-dev/positron#16137). + filtering, profiles, export and Convert to Code. The kernel serves this + through a frontend-neutral results API, reached with `call_method` on the + UI comm and turned on by the frontend (`--keep-results`), which keeps + results in an in-memory cache of its own within one byte budget + (`--results-max-bytes`, default 1 GB) and never changes the session's + reader or dialect (#595). ### Changed @@ -48,6 +48,10 @@ determines whether text boxes get drawn and additional settings `padding`, - The CLI now always prints query warnings (e.g. extra `VISUALISE`/`TABULATE` statements being ignored) to stderr, not only with `-v`. - Replaced `CLAUDE.md` files with `AGENTS.md` files. +- A Positron console now shows a long query result's first 10 rows, followed + by a count of the rest, rather than up to 100; a result of up to 20 rows is + still shown whole. The number is set by the `ggsql.consoleRows` setting + (#595). ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 6968a7332..3cdb9db58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1791,7 +1791,6 @@ dependencies = [ "ggsql", "hex", "hmac", - "regex", "serde", "serde_json", "sha2", diff --git a/Cargo.toml b/Cargo.toml index b9e1bd2df..457247270 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ tree-sitter = "0.26" arrow = { version = "58", default-features = false } # Readers -duckdb = { version = "~1.10502", features = ["bundled", "vtab-arrow", "appender-arrow"] } +duckdb = { version = "~1.10502", features = ["bundled", "vtab-arrow"] } parquet = { version = "58", default-features = false, features = ["arrow", "snap"] } bytes = "1" rusqlite = { version = "0.38", features = ["bundled", "chrono", "load_extension"] } diff --git a/doc/get_started/tooling/positron-vscode.qmd b/doc/get_started/tooling/positron-vscode.qmd index 0aa5c828e..4662f1a41 100644 --- a/doc/get_started/tooling/positron-vscode.qmd +++ b/doc/get_started/tooling/positron-vscode.qmd @@ -34,7 +34,7 @@ Executing ggsql statements containing [`VISUALISE`](../../syntax/clause/visualis A query without `VISUALISE` shows its result as a table in the console, followed by an "Open in Data Explorer" link. A result of more than 20 rows shows only its first 10, with a count of the rest. To show more or fewer, change the `ggsql.consoleRows` setting: a result of up to twice that many rows is shown in full. The change applies to consoles started afterwards. Clicking the link opens the result in Positron's Data Explorer, where you can sort, filter, and profile it, and use "Convert to Code" to turn your sorts and filters back into SQL. -The Data Explorer works on a snapshot of the result, so it doesn't change when the underlying tables do. Run the query again to see new data. The console keeps the last 16 results available this way; the link under an older one reports that it is no longer available. +The Data Explorer works on a snapshot of the result, so it doesn't change when the underlying tables do. Run the query again to see new data. The console keeps recent results available this way, up to 1 GB in all. Results open in a Data Explorer are always kept, and older ones make way for new ones, so the link under an old result may report that it is no longer available. If the results you have open use the whole 1 GB, a new result is shown in the console without a link, and sorting or filtering a large one may ask you to close another Data Explorer tab first. ## Editor and cells diff --git a/ggsql-jupyter/AGENTS.md b/ggsql-jupyter/AGENTS.md index 0b7aa6bd1..1c2714760 100644 --- a/ggsql-jupyter/AGENTS.md +++ b/ggsql-jupyter/AGENTS.md @@ -185,16 +185,20 @@ So `RenderHints::canvas` picks a height (golden ratio, close to ggplot2's defaul ### The results API -`results/` keeps recent **console** query results and answers a small, frontend-neutral set of calls on them (`methods.rs` documents each: `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`). Nothing in it knows about Positron: the ggsql extension translates Positron's Data Explorer protocol into these calls (see [`ggsql-vscode/AGENTS.md`](../ggsql-vscode/AGENTS.md)). The kernel's only Positron-specific part is the route: a `call_method` request on the `positron.ui` comm — what an extension's `session.callMethod(name, params)` sends — whose method starts with `result_` goes to `methods::call`, and its answer or error comes back as the JSON-RPC reply (errors with `plot::comm::RpcError`'s encoding). Any other UI request still gets `result: null`. +`results/` keeps recent query results and answers a small, frontend-neutral set of calls on them (`methods.rs` documents each: `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`). Nothing in it knows about Positron: the ggsql extension translates Positron's Data Explorer protocol into these calls (see [`ggsql-vscode/AGENTS.md`](../ggsql-vscode/AGENTS.md)). The kernel's only Positron-specific part is the route: a `call_method` request on the `positron.ui` comm — what an extension's `session.callMethod(name, params)` sends — whose method starts with `result_` goes to `methods::call`, and its answer or error comes back as the JSON-RPC reply (errors with `plot::comm::RpcError`'s encoding). A `call_method` for any other name is an error, like an unknown plot-comm method, so a caller expecting something the kernel lacks fails visibly; any other UI request gets `result: null`. -- **Which results are kept.** A tabular result of a non-silent execution in a `PositronConsole` session. Its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data, the newest `--max-results` (default 16). -- **Opening copies the result into the results cache.** A `CacheTables` (the caching reader's bookkeeping, `ggsql::reader::cache`) on its own in-memory DuckDB, bounded by `GGSQL_RESULTS_MAX_BYTES` (default 1 GB). The session's reader is never wrapped, so the user's dialect is untouched. The copy adds `__ggsql_row__`, each row's position in the query's output, and drops the Arrow data, so a result is held once. It stays **pinned** while any view of it is open, so eviction never pulls a table from under an open view; once closed, it is evicted when the cache needs the room, and reopening before then reuses it. +- **Which results are kept.** Only when the frontend asks, with `--keep-results` or `GGSQL_KEEP_RESULTS` — the extension sets it for a console session whose results it can open — so a session nobody explores holds nothing beyond what it displays. Then every tabular result of a non-silent execution is kept, and its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data. +- **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key) and a sort or filter that needs room fails with an error saying so. DuckDB's own working memory during a sort comes on top of the estimate. +- **Opening copies the result into the results cache.** A `CacheTables` (the caching reader's bookkeeping, `ggsql::reader::cache`) on its own in-memory DuckDB, which `ResultStore` creates itself. The session's reader is never wrapped, so the user's dialect is untouched. The copy adds `__ggsql_row__`, each row's position in the query's output, and drops the Arrow data, so a result is held once. It stays **pinned** while any view of it is open; once closed, it goes when the budget needs the room, and reopening before then reuses it. A result can have several views, each with its own filters and sort. +- **Views end with the UI comm.** Closing it closes every view, since nothing could close them once it has gone; that is what releases the pins of a frontend that reloaded or crashed. - **Nothing is written to disk.** The cache sets `temp_directory = ''`, so DuckDB cannot spill: a query that needs more memory than the limit fails with an error instead. -- **Sorting and filtering build an order table** of `(pos, __ggsql_row__)` per view (`view.rs`); a page joins it back to the result by `pos` range, so scrolling never sorts again. Ties break on `__ggsql_row__`, so paging is stable and clearing the sort restores the query's own order. An unfiltered, unsorted view has none and pages by `__ggsql_row__` directly. -- **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. -- **`result_sql` writes SQL for the session's database**, not the cache: the result's query as a derived table, with the view's filters and sort, in the session's dialect. A filter it cannot express there is left out with a `--` comment. +- **Sorting and filtering build an order table** of `(pos, __ggsql_row__)` per view (`view.rs`); a page joins it back to the result by `pos`, so scrolling never sorts again. Ties break on `__ggsql_row__`, so paging is stable and clearing the sort restores the query's own order. An unfiltered, unsorted view has none and pages by `__ggsql_row__` directly. A new sort or set of filters is built under a new table name and only then replaces the old one, so one that fails leaves the view as it was. +- **A page is bounded.** At most `MAX_PAGE_ROWS`; a range is cut short at the end of the view, and an index outside it is an error, so a cell never lands in another row's place. Scattered indices fetch just those rows. +- **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, booleans and regular expressions come from the dialect the SQL is for (`SqlDialect::sql_regex_match`, `None` where a backend has none), and `LIKE` escapes with `!`, which no backend reads as a string escape. +- **Kinds are DuckDB type names matched exactly** (`kind_of`): a column type that only contains `INT`, like `POINT`, is not an integer, and an `ENUM` is a string. +- **`result_sql` writes SQL for the session's database**, not the cache: the cell's statements before the last as they were, then the last as a derived table with the view's filters and sort, in the session's dialect. It is in the kernel because only the kernel knows that dialect. A filter it cannot express there, or one on a column whose name the query repeats (renamed `a_1` in the cache, but ambiguous in SQL), is left out with a `--` comment. -`examples/results_bench.rs` measures it. At 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 1.5 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. +`examples/results_bench.rs` measures it. At 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 3.4 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. ## Build & install diff --git a/ggsql-jupyter/Cargo.toml b/ggsql-jupyter/Cargo.toml index 1774a883d..4cb194267 100644 --- a/ggsql-jupyter/Cargo.toml +++ b/ggsql-jupyter/Cargo.toml @@ -39,9 +39,6 @@ clap = { workspace = true, features = ["env"] } serde = { workspace = true } serde_json = { workspace = true } -# The data explorer's regular-expression column search -regex = { workspace = true } - # Error handling anyhow = { workspace = true } thiserror = { workspace = true } diff --git a/ggsql-jupyter/examples/results_bench.rs b/ggsql-jupyter/examples/results_bench.rs index 37492be1b..ee236bda2 100644 --- a/ggsql-jupyter/examples/results_bench.rs +++ b/ggsql-jupyter/examples/results_bench.rs @@ -20,7 +20,6 @@ use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; use ggsql::reader::connection::reader_from_uri; use ggsql::DataFrame; -use ggsql_jupyter::results::filter::Capabilities; use ggsql_jupyter::results::methods::{self, Session}; use ggsql_jupyter::results::ResultStore; use serde_json::{json, Value}; @@ -169,14 +168,17 @@ fn run_store(batch: RecordBatch, cycles: usize) { let session_reader = reader_from_uri("duckdb://memory").unwrap(); let session = Session { reader: &*session_reader, - caps: Capabilities { regex: true }, }; - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert( - DataFrame::from_record_batch(batch), - "bench".into(), - "SELECT 1".into(), - ); + // Unbounded, so the benchmark measures the work rather than the budget. + let mut store = ResultStore::new(u64::MAX).unwrap(); + let key = store + .insert( + DataFrame::from_record_batch(batch), + "bench".into(), + "SELECT 1".into(), + ) + .unwrap() + .expect("an unbounded store keeps every result"); let mut call = |method: &str, params: Value| { methods::call(&mut store, &session, method, ¶ms) .unwrap_or_else(|e| panic!("{method}: {e}")) diff --git a/ggsql-jupyter/src/display.rs b/ggsql-jupyter/src/display.rs index 37ce7774b..85ddb2fdb 100644 --- a/ggsql-jupyter/src/display.rs +++ b/ggsql-jupyter/src/display.rs @@ -276,7 +276,12 @@ fn format_static( /// Format DataFrame as HTML table fn format_dataframe(df: DataFrame, hints: &RenderHints) -> Value { let html = dataframe_to_html(&df, html_rows_shown(df.height(), hints)); - let text = dataframe_to_text(&df); + // The plain text is the console's too, where it follows the same rule. + let text_rows = match hints.kind { + SessionKind::PositronConsole => html_rows_shown(df.height(), hints), + _ => df.height().min(10), + }; + let text = dataframe_to_text(&df, text_rows); json!({ "data": { @@ -350,12 +355,11 @@ fn dataframe_to_html(df: &DataFrame, row_limit: usize) -> String { } /// Convert DataFrame to plain-text summary (shape + column names + first rows). -fn dataframe_to_text(df: &ggsql::DataFrame) -> String { +fn dataframe_to_text(df: &ggsql::DataFrame, row_limit: usize) -> String { let mut s = format!("shape: ({}, {})\n", df.height(), df.width()); let names = df.get_column_names(); s.push_str(&names.join("\t")); s.push('\n'); - let row_limit = df.height().min(10); for i in 0..row_limit { let row: Vec = df.get_columns().iter().map(|c| cell_text(c, i)).collect(); s.push_str(&row.join("\t")); @@ -402,7 +406,7 @@ mod tests { .unwrap(); let html = dataframe_to_html(&df, 1); assert!(html.contains("2024-01-01"), "{html}"); - assert!(dataframe_to_text(&df).contains("2024-01-01")); + assert!(dataframe_to_text(&df, 1).contains("2024-01-01")); } #[test] @@ -594,6 +598,22 @@ mod tests { assert!(html.contains("... 4990 more rows"), "{html}"); } + #[test] + fn test_console_text_follows_console_rows() { + let df = ggsql::reader::connection::reader_from_uri("duckdb://memory") + .unwrap() + .execute_sql("SELECT i FROM range(50) t(i)") + .unwrap(); + let console = RenderHints { + console_rows: 3, + ..positron_console() + }; + let bundle = format_dataframe(df, &console); + let text = bundle["data"]["text/plain"].as_str().unwrap(); + // The shape line, the header, and three rows. + assert_eq!(text.lines().count(), 2 + 3, "{text}"); + } + /// A render backend with no GPU, so tests are fast and identical /// everywhere. The SVG path it leaves is the one that always works. fn backend() -> PlotBackend { diff --git a/ggsql-jupyter/src/kernel.rs b/ggsql-jupyter/src/kernel.rs index bc9f9db6e..a88c7eed9 100644 --- a/ggsql-jupyter/src/kernel.rs +++ b/ggsql-jupyter/src/kernel.rs @@ -5,12 +5,11 @@ use crate::connection; use crate::data_explorer::{DataExplorerState, RpcResponse}; -use crate::display::{format_display_data, Formatted, RenderHints, SessionKind, SessionMode}; +use crate::display::{format_display_data, Formatted, RenderHints, SessionMode}; use crate::executor::{self, ExecutionResult, QueryExecutor}; use crate::message::{ConnectionInfo, JupyterMessage, MessageHeader}; use crate::plot::comm::{PlotMetadata, RenderParams, RpcError}; use crate::plot::{PlotBackend, RenderOutcome, RenderTicket}; -use crate::results::filter::Capabilities; use crate::results::{methods, ResultStore}; use anyhow::Result; use ggsql::reader::ResolvedPlot; @@ -67,8 +66,11 @@ pub struct KernelServer { ui_comm_id: Option, connection_comm_id: Option, data_explorer_comms: HashMap, - /// Console query results kept for the results API. + /// Query results kept for the results API. results: ResultStore, + /// `--keep-results`: whether to keep results for the results API, which + /// the frontend that will explore them turns on. + keep_results: bool, /// `--console-rows`: how many rows of a long result a console shows. console_rows: usize, } @@ -80,7 +82,8 @@ impl KernelServer { reader_uri: &str, session_mode: Option, max_plots: usize, - max_results: usize, + results_max_bytes: u64, + keep_results: bool, console_rows: usize, ) -> Result { tracing::info!("Initializing kernel server"); @@ -149,7 +152,8 @@ impl KernelServer { ui_comm_id: None, connection_comm_id: None, data_explorer_comms: HashMap::new(), - results: ResultStore::new(max_results)?, + results: ResultStore::new(results_max_bytes)?, + keep_results, console_rows, }; @@ -417,21 +421,26 @@ impl KernelServer { self.open_connection_comm(&uri).await?; } - // Formatting renders the plot, so it is fallible and its failures are - // the user's. Both stages feed one `Result` and share the error path - // below; propagating out of the loop would end the process and take - // the session's database with it. - // A console result is kept for the results API, and its key goes in - // the `execute_result`'s metadata so a frontend can offer to explore - // it. A silent cell shows nothing to offer, and DDL has no table. + // A result is kept for the results API when the frontend asked for + // that, and its key goes in the `execute_result`'s metadata so the + // frontend can offer to explore it. A silent cell shows nothing to + // offer, and DDL has no table. A result that does not fit the budget + // is shown but not kept. let mut retained = None; if let Ok(ExecutionResult::DataFrame(df)) = &executed { - if !silent && hints.kind == SessionKind::PositronConsole && df.width() > 0 { + if self.keep_results && !silent && df.width() > 0 { let title = format!("Query [{}]", self.execution_count); - retained = Some(self.results.insert(df.clone(), title, query_of(code))); + match self.results.insert(df.clone(), title, query_of(code)) { + Ok(key) => retained = key, + Err(e) => tracing::warn!("Keeping a result failed: {}", e), + } } } + // Formatting renders the plot, so it is fallible and its failures are + // the user's. Both stages feed one `Result` and share the error path + // below; propagating out of the loop would end the process and take + // the session's database with it. let result = match executed { Ok(exec_result) => { // A bare connection change renders nothing. @@ -753,9 +762,6 @@ impl KernelServer { let answer = if methods::handles(name) { let session = methods::Session { reader: self.executor.reader(), - caps: Capabilities { - regex: self.executor.reader_uri().starts_with("duckdb://"), - }, }; methods::call(&mut self.results, &session, name, args) } else { @@ -920,6 +926,9 @@ impl KernelServer { } else if Some(comm_id.to_string()) == self.ui_comm_id { tracing::info!("Closing positron.ui comm"); self.ui_comm_id = None; + // The views were opened over this comm, and nothing can close + // them once it has gone. + self.results.close_all(); } else if self.plot_comms.remove(comm_id).is_some() { tracing::info!("Closing positron.plot comm {}", comm_id); self.plot_order.retain(|id| id != comm_id); diff --git a/ggsql-jupyter/src/main.rs b/ggsql-jupyter/src/main.rs index a99f1d9d7..62ded8fb2 100644 --- a/ggsql-jupyter/src/main.rs +++ b/ggsql-jupyter/src/main.rs @@ -49,14 +49,33 @@ struct Args { #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u32).range(1..))] max_plots: u32, - /// How many query results to keep available for the data explorer. + /// Keep query results so a frontend can explore them through the results + /// API. /// - /// In a Positron console each tabular result is kept for the results API, - /// so a frontend can open it in a data explorer later. This bounds the - /// results not opened yet, dropping the oldest; opened results are bounded - /// by `GGSQL_RESULTS_MAX_BYTES` instead (default 1 GB). - #[arg(long, default_value_t = 16, value_parser = clap::value_parser!(u32).range(1..))] - max_results: u32, + /// Off unless the frontend that will explore them turns it on, so a + /// session nobody explores holds nothing beyond what it displays. Also + /// read from the environment, which is how the ggsql extension sets it. + #[arg( + long, + env = "GGSQL_KEEP_RESULTS", + value_parser = clap::builder::FalseyValueParser::new() + )] + keep_results: bool, + + /// The most memory kept results may use, e.g. `512mb` or `2gb`. + /// + /// Counts results not opened yet, opened ones and the order tables of + /// their sorted or filtered views, by the size of their data. Closed + /// results go first when room is needed, then the oldest unopened ones; + /// open ones are never dropped, so a result that does not fit beside them + /// is not kept. + #[arg( + long, + env = "GGSQL_RESULTS_MAX_BYTES", + default_value_t = results::DEFAULT_MAX_BYTES, + value_parser = parse_bytes + )] + results_max_bytes: u64, /// How many rows of a query result a Positron console shows. /// @@ -85,6 +104,13 @@ struct Args { sys_prefix: bool, } +/// A byte count, with an optional `kb`/`mb`/`gb` suffix. +fn parse_bytes(s: &str) -> std::result::Result { + ggsql::reader::cache::parse_human_bytes(s) + .filter(|&n| n > 0) + .ok_or_else(|| format!("{s:?} is not a byte count, such as 1gb")) +} + #[tokio::main] async fn main() -> Result<()> { // Initialize tracing @@ -124,7 +150,8 @@ async fn main() -> Result<()> { &args.reader, args.session_mode, args.max_plots as usize, - args.max_results as usize, + args.results_max_bytes, + args.keep_results, args.console_rows as usize, ) .await?; diff --git a/ggsql-jupyter/src/results/filter.rs b/ggsql-jupyter/src/results/filter.rs index ae2ea05e9..34969266f 100644 --- a/ggsql-jupyter/src/results/filter.rs +++ b/ggsql-jupyter/src/results/filter.rs @@ -1,20 +1,17 @@ //! Row filters and sort keys over a result's columns, and the SQL for them. //! -//! Everything here builds from the request and the result's schema, with no -//! database, so it is tested as plain string assembly. +//! Everything here builds from the request, the result's schema and a +//! [`SqlDialect`], with no database, so it is tested as plain string assembly. +//! The same code writes SQL for the results cache, to filter and sort a view, +//! and for the session's own database, for `result_sql`: identifiers, +//! booleans and regular expressions all come from the dialect it is given. use super::Column; -use ggsql::naming::{quote_ident, quote_literal}; +use ggsql::naming::quote_literal; +use ggsql::parser::SourceTree; use ggsql::reader::SqlDialect; use serde::{Deserialize, Serialize}; -/// What a database can do beyond the SQL every backend shares. -#[derive(Debug, Clone, Copy)] -pub struct Capabilities { - /// `regexp_matches`, for regular-expression text filters. DuckDB only. - pub regex: bool, -} - /// A comparison operator. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CompareOp { @@ -130,10 +127,9 @@ pub fn predicate( filter: &Filter, columns: &[Column], dialect: &dyn SqlDialect, - caps: Capabilities, ) -> Result { let column = column_of(columns, filter.column())?; - let c = quote_ident(&column.name); + let c = dialect.quote_ident(&column.name); let lit = |v: &str| literal(v, &column.kind, dialect); match filter { @@ -169,7 +165,7 @@ pub fn predicate( term, case_sensitive, .. - } => text(&c, *how, term, *case_sensitive, caps), + } => text(&c, *how, term, *case_sensitive, dialect), Filter::In { values, negate, .. } => { if values.is_empty() { // Nothing selected: keep nothing, or exclude nothing. @@ -214,17 +210,12 @@ fn text( how: TextMatch, term: &str, case_sensitive: bool, - caps: Capabilities, + dialect: &dyn SqlDialect, ) -> Result { if how == TextMatch::Regex { - if !caps.regex { - return Err("This database does not support regular-expression search".into()); - } - let flags = if case_sensitive { "" } else { ", 'i'" }; - return Ok(format!( - "regexp_matches({c}, {}{flags})", - quote_literal(term) - )); + return dialect + .sql_regex_match(c, "e_literal(term), case_sensitive) + .ok_or_else(|| "This database does not support regular-expression search".into()); } let escaped = escape_like(term); @@ -244,55 +235,99 @@ fn text( ) }; let not = if negate { "NOT " } else { "" }; - Ok(format!("{lhs} {not}LIKE {rhs} ESCAPE '\\'")) + Ok(format!("{lhs} {not}LIKE {rhs} ESCAPE '{LIKE_ESCAPE}'")) } +/// `LIKE`'s escape character. Not a backslash, which MySQL and Snowflake +/// read as an escape inside the string literal itself, so `'\'` would never +/// end there. +const LIKE_ESCAPE: char = '!'; + /// Escape `LIKE`'s wildcards, so a search term matches literally. fn escape_like(term: &str) -> String { - term.replace('\\', "\\\\") - .replace('%', "\\%") - .replace('_', "\\_") + let mut out = String::with_capacity(term.len()); + for ch in term.chars() { + if matches!(ch, '%' | '_') || ch == LIKE_ESCAPE { + out.push(LIKE_ESCAPE); + } + out.push(ch); + } + out } /// The sort keys as `ORDER BY` terms, without the keyword. -pub fn order_terms(keys: &[SortKey], columns: &[Column]) -> Result, String> { +pub fn order_terms( + keys: &[SortKey], + columns: &[Column], + dialect: &dyn SqlDialect, +) -> Result, String> { keys.iter() .map(|key| { let column = column_of(columns, key.column)?; let direction = if key.ascending { "ASC" } else { "DESC" }; - Ok(format!("{} {direction}", quote_ident(&column.name))) + Ok(format!("{} {direction}", dialect.quote_ident(&column.name))) }) .collect() } -/// SQL reproducing a view of a result: `from` filtered and sorted. +/// Why `column` cannot be named in SQL over the query, if it cannot. +fn unnameable(columns: &[Column], index: usize) -> Option { + let column = columns.get(index)?; + column.repeated.then(|| { + format!( + "column {:?} repeats a name in the query, which SQL cannot refer to", + column.name + ) + }) +} + +/// SQL for `dialect` reproducing a view of the result of `query`: its last +/// statement, filtered and sorted, after the statements that come before it. /// -/// A filter that cannot be expressed in `dialect` is left out with a comment -/// saying so, rather than failing the whole conversion. +/// A filter or sort key that cannot be expressed in `dialect` is left out with +/// a comment saying so, rather than failing the whole conversion. pub fn to_sql( filters: &[Filter], sort: &[SortKey], - from: &str, + query: &str, columns: &[Column], dialect: &dyn SqlDialect, - caps: Capabilities, ) -> Vec { let mut lines = Vec::new(); let mut predicates = Vec::new(); for filter in filters { - match predicate(filter, columns, dialect, caps) { + let built = match unnameable(columns, filter.column()) { + Some(why) => Err(why), + None => predicate(filter, columns, dialect), + }; + match built { Ok(p) => predicates.push(p), Err(e) => lines.push(format!("-- Filter left out: {e}")), } } + let mut keys = Vec::new(); + for key in sort { + match unnameable(columns, key.column) { + Some(why) => lines.push(format!("-- Sort key left out: {why}")), + None => keys.push(key.clone()), + } + } + let (setup, last) = split_statements(query); + for stmt in setup { + lines.extend(format!("{stmt};").lines().map(str::to_string)); + } lines.push("SELECT *".to_string()); - lines.extend(format!("FROM {from}").lines().map(str::to_string)); + lines.extend( + format!("FROM {}", subquery(&last, dialect)) + .lines() + .map(str::to_string), + ); for (i, p) in predicates.iter().enumerate() { let keyword = if i == 0 { "WHERE" } else { " AND" }; lines.push(format!("{keyword} {p}")); } - match order_terms(sort, columns) { + match order_terms(&keys, columns, dialect) { Ok(terms) if !terms.is_empty() => lines.push(format!("ORDER BY {}", terms.join(", "))), Ok(_) => {} Err(e) => lines.push(format!("-- Sort left out: {e}")), @@ -300,20 +335,55 @@ pub fn to_sql( lines } +/// A cell's statements: those before the last, and the last, which is the one +/// whose result was kept. A query that does not parse is one statement. +fn split_statements(query: &str) -> (Vec, String) { + let whole = || (Vec::new(), query.trim().trim_end_matches(';').to_string()); + let Ok(tree) = SourceTree::new(query) else { + return whole(); + }; + let root = tree.root(); + let mut statements: Vec = tree + .find_nodes(&root, "(sql_statement) @s") + .into_iter() + // The outermost statements: a subquery can be a statement of its own. + .filter(|node| { + let mut up = node.parent(); + while let Some(p) = up { + if p.kind() == "sql_statement" { + return false; + } + up = p.parent(); + } + true + }) + .map(|node| tree.get_text(&node).trim().to_string()) + .filter(|text| !text.is_empty()) + .collect(); + match statements.pop() { + Some(last) => (statements, last), + None => whole(), + } +} + /// A query as a derived table, for wrapping in a view's filters. -pub fn subquery(query: &str) -> String { +fn subquery(query: &str, dialect: &dyn SqlDialect) -> String { let query = query.trim().trim_end_matches(';').trim_end(); let body: Vec = query.lines().map(|l| format!(" {l}")).collect(); - format!("(\n{}\n) AS {}", body.join("\n"), quote_ident("result")) + format!( + "(\n{}\n) {}", + body.join("\n"), + dialect.sql_table_alias("result") + ) } #[cfg(test)] mod tests { use super::*; + use ggsql::reader::dialects::DuckDbDialect; + use ggsql::reader::registry::dialect_override as dialect_for_scheme; use ggsql::reader::AnsiDialect; - use serde_json::json; - - const CAPS: Capabilities = Capabilities { regex: true }; + use serde_json::{json, Value}; fn columns() -> Vec { [("n", "integer"), ("name", "string"), ("ok", "boolean")] @@ -322,17 +392,16 @@ mod tests { name: name.to_string(), type_name: String::new(), kind: kind.to_string(), + repeated: false, }) .collect() } fn sql(f: Value) -> Result { let filter: Filter = serde_json::from_value(f).unwrap(); - predicate(&filter, &columns(), &AnsiDialect, CAPS) + predicate(&filter, &columns(), &AnsiDialect) } - use serde_json::Value; - #[test] fn test_compare_maps_not_equal_and_checks_numbers() { assert_eq!( @@ -365,8 +434,9 @@ mod tests { #[test] fn test_text_escapes_wildcards_and_folds_case() { assert_eq!( - sql(json!({"type": "text", "column": 1, "match": "contains", "term": "50%_"})).unwrap(), - "LOWER(\"name\") LIKE LOWER('%50\\%\\_%') ESCAPE '\\'" + sql(json!({"type": "text", "column": 1, "match": "contains", "term": "50%_!"})) + .unwrap(), + "LOWER(\"name\") LIKE LOWER('%50!%!_!!%') ESCAPE '!'" ); assert_eq!( sql( @@ -374,21 +444,37 @@ mod tests { "case_sensitive": true}) ) .unwrap(), - "\"name\" LIKE 'a%' ESCAPE '\\'" + "\"name\" LIKE 'a%' ESCAPE '!'" ); } #[test] - fn test_regex_needs_the_capability() { + fn test_the_like_escape_matches_literally_in_duckdb() { + let reader = ggsql::reader::connection::reader_from_uri("duckdb://memory").unwrap(); + let filter: Filter = serde_json::from_value( + json!({"type": "text", "column": 1, "match": "contains", "term": "5%!_"}), + ) + .unwrap(); + let p = predicate(&filter, &columns(), &DuckDbDialect).unwrap(); + let df = reader + .execute_sql(&format!( + "SELECT name FROM (VALUES ('a5%!_b'), ('a5x!_b'), ('a5%!xb')) t(name) WHERE {p}" + )) + .unwrap(); + assert_eq!(df.height(), 1); + } + + #[test] + fn test_regex_comes_from_the_dialect() { let f: Filter = serde_json::from_value( json!({"type": "text", "column": 1, "match": "regex", "term": "^a"}), ) .unwrap(); assert_eq!( - predicate(&f, &columns(), &AnsiDialect, CAPS).unwrap(), + predicate(&f, &columns(), &DuckDbDialect).unwrap(), "regexp_matches(\"name\", '^a', 'i')" ); - assert!(predicate(&f, &columns(), &AnsiDialect, Capabilities { regex: false }).is_err()); + assert!(predicate(&f, &columns(), &AnsiDialect).is_err()); } #[test] @@ -420,25 +506,35 @@ mod tests { column: 9, ascending: true }], - &columns() + &columns(), + &AnsiDialect ) .is_err()); } + fn filters(f: Value) -> Vec { + serde_json::from_value(f).unwrap() + } + + const BY_N: [SortKey; 1] = [SortKey { + column: 0, + ascending: true, + }]; + #[test] fn test_to_sql_wraps_the_query() { - let filters: Vec = serde_json::from_value(json!([ + let filters = filters(json!([ {"type": "compare", "column": 0, "op": ">", "value": "2"}, {"type": "is_null", "column": 1} - ])) - .unwrap(); - let sort = [SortKey { - column: 0, - ascending: true, - }]; - let from = subquery("SELECT *\nFROM t;"); + ])); assert_eq!( - to_sql(&filters, &sort, &from, &columns(), &AnsiDialect, CAPS), + to_sql( + &filters, + &BY_N, + "SELECT *\nFROM t;", + &columns(), + &AnsiDialect + ), vec![ "SELECT *", "FROM (", @@ -452,21 +548,78 @@ mod tests { ); } + #[test] + fn test_to_sql_quotes_for_the_session_dialect() { + let filters = filters(json!([ + {"type": "text", "column": 1, "match": "contains", "term": "x"} + ])); + for scheme in ["mysql", "bigquery", "clickhouse", "databricks"] { + let dialect = dialect_for_scheme(scheme).unwrap(); + let sql = to_sql(&filters, &BY_N, "SELECT 1", &columns(), dialect).join("\n"); + assert!( + sql.contains("LOWER(`name`) LIKE LOWER('%x%') ESCAPE '!'"), + "{scheme}: {sql}" + ); + assert!(sql.contains("ORDER BY `n` ASC"), "{scheme}: {sql}"); + assert!(!sql.contains("\"name\""), "{scheme}: {sql}"); + } + } + + #[test] + fn test_to_sql_keeps_the_statements_before_the_query() { + let query = "CREATE TABLE t AS SELECT 1 AS n;\nINSERT INTO t VALUES (2);\nSELECT * FROM t"; + assert_eq!( + to_sql(&[], &BY_N, query, &columns(), &AnsiDialect), + vec![ + "CREATE TABLE t AS SELECT 1 AS n;", + "INSERT INTO t VALUES (2);", + "SELECT *", + "FROM (", + " SELECT * FROM t", + ") AS \"result\"", + "ORDER BY \"n\" ASC", + ] + ); + // And what it writes runs. + let reader = ggsql::reader::connection::reader_from_uri("duckdb://memory").unwrap(); + let lines = to_sql(&[], &BY_N, query, &columns(), &DuckDbDialect).join("\n"); + let mut last = None; + for stmt in lines.split(';') { + last = Some(reader.execute_sql(stmt).unwrap()); + } + assert_eq!(last.unwrap().height(), 2); + } + + #[test] + fn test_to_sql_leaves_out_a_repeated_column() { + let mut cols = columns(); + cols[1].repeated = true; + let filters = filters(json!([{"type": "is_null", "column": 1}])); + let sort = [SortKey { + column: 1, + ascending: true, + }]; + let lines = to_sql(&filters, &sort, "SELECT 1", &cols, &AnsiDialect); + assert!( + lines[0].starts_with("-- Filter left out: column \"name\" repeats"), + "{lines:?}" + ); + assert!(lines[1].starts_with("-- Sort key left out"), "{lines:?}"); + assert!(!lines + .iter() + .any(|l| l.starts_with("WHERE") || l.starts_with("ORDER"))); + } + #[test] fn test_to_sql_leaves_out_what_it_cannot_express() { - let filters: Vec = serde_json::from_value(json!([ + let filters = filters(json!([ {"type": "text", "column": 1, "match": "regex", "term": "x", "case_sensitive": true} - ])) - .unwrap(); - let lines = to_sql( - &filters, - &[], - "\"t\"", - &columns(), - &AnsiDialect, - Capabilities { regex: false }, - ); + ])); + let lines = to_sql(&filters, &[], "SELECT 1", &columns(), &AnsiDialect); assert!(lines[0].starts_with("-- Filter left out"), "{lines:?}"); - assert_eq!(&lines[1..], ["SELECT *", "FROM \"t\""]); + assert_eq!( + &lines[1..], + ["SELECT *", "FROM (", " SELECT 1", ") AS \"result\""] + ); } } diff --git a/ggsql-jupyter/src/results/methods.rs b/ggsql-jupyter/src/results/methods.rs index d961e2aa9..ae19319b9 100644 --- a/ggsql-jupyter/src/results/methods.rs +++ b/ggsql-jupyter/src/results/methods.rs @@ -13,8 +13,19 @@ //! //! `num_rows` after filtering counts the rows that pass. A filter that cannot //! be applied is left out and its message returned at its index in `errors`. +//! A sort or set of filters that cannot be applied at all is an error, and +//! leaves the view as it was. +//! +//! A page holds at most [`MAX_PAGE_ROWS`] rows. A range is cut short at the end +//! of the view; an index outside it is an error. +//! +//! `result_sql` is here rather than in a frontend because only the kernel +//! knows the session's database: it writes the view as SQL for that dialect, +//! which a frontend could not do from the neutral calls alone. +//! +//! [`MAX_PAGE_ROWS`]: super::view::MAX_PAGE_ROWS -use super::filter::{Capabilities, Filter, SortKey}; +use super::filter::{Filter, SortKey}; use super::view::{ProfileRequest, Rows}; use super::ResultStore; use ggsql::reader::Reader; @@ -30,12 +41,8 @@ pub fn handles(method: &str) -> bool { /// The session's database, which `result_sql` writes SQL for. pub struct Session<'a> { pub reader: &'a dyn Reader, - pub caps: Capabilities, } -/// The results cache is DuckDB, which has `regexp_matches`. -const CACHE_CAPS: Capabilities = Capabilities { regex: true }; - #[derive(Deserialize)] struct Open { key: String, @@ -106,14 +113,13 @@ pub fn call( } "result_set_filters" => { let p: SetFilters = parse(params)?; - let (view, cache) = store.view_mut(&p.view).map_err(err)?; - let errors = view.set_filters(p.filters, cache.backend(), CACHE_CAPS)?; + let errors = store.set_filters(&p.view, p.filters)?; + let (view, _) = store.view_mut(&p.view).map_err(err)?; Ok(json!({ "num_rows": view.selected_rows, "errors": errors })) } "result_set_sort" => { let p: SetSort = parse(params)?; - let (view, cache) = store.view_mut(&p.view).map_err(err)?; - view.set_sort(p.keys, cache.backend())?; + store.set_sort(&p.view, p.keys)?; Ok(json!({})) } "result_page" => { @@ -137,7 +143,7 @@ pub fn call( let p: ViewOnly = parse(params)?; let query = store.query_of(&p.view).map_err(err)?.to_string(); let (view, _) = store.view_mut(&p.view).map_err(err)?; - let sql = view.to_sql(&query, session.reader.dialect(), session.caps); + let sql = view.to_sql(&query, session.reader.dialect()); Ok(json!({ "sql": sql })) } "result_close" => { @@ -159,18 +165,18 @@ mod tests { let df = session .execute_sql("SELECT i AS n, 'x' || i AS s FROM range(5) t(i)") .unwrap(); - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(df, "Query [1]".into(), "SELECT * FROM t;".into()); + let mut store = ResultStore::new(super::super::DEFAULT_MAX_BYTES).unwrap(); + let key = store + .insert(df, "Query [1]".into(), "SELECT * FROM t;".into()) + .unwrap() + .unwrap(); (store, session, key) } #[test] fn test_a_round_trip_through_the_calls() { let (mut store, reader, key) = setup(); - let session = Session { - reader: &*reader, - caps: Capabilities { regex: true }, - }; + let session = Session { reader: &*reader }; let mut call = |m: &str, p: Value| call(&mut store, &session, m, &p); // Positional arguments, as `callMethod` sends them. @@ -217,6 +223,22 @@ mod tests { ] ); + let profile = call( + "result_profile", + json!({"view": view, "column": 0, "profiles": {"null_count": true, "summary": true}}), + ) + .unwrap(); + assert_eq!(profile["null_count"], 0); + assert_eq!(profile["summary"]["min"], "2"); + assert_eq!(profile["summary"]["max"], "4"); + + let export = call( + "result_export", + json!({"view": view, "rows": {"indices": [1, 0]}, "columns": [0, 1], "format": "tsv"}), + ) + .unwrap(); + assert_eq!(export["data"], "n\ts\n3\tx3\n4\tx4"); + assert_eq!( call("result_close", json!({ "view": view })).unwrap(), json!({}) @@ -229,13 +251,33 @@ mod tests { .contains("no view")); } + #[test] + fn test_the_session_database_is_untouched() { + let (mut store, reader, key) = setup(); + let session = Session { reader: &*reader }; + let tables = || { + reader + .execute_sql("SELECT table_name FROM duckdb_tables()") + .unwrap() + .height() + }; + let before = tables(); + let opened = call(&mut store, &session, "result_open", &json!({ "key": key })).unwrap(); + let view = opened["view"].as_str().unwrap(); + call( + &mut store, + &session, + "result_set_sort", + &json!({"view": view, "keys": [{"column": 0, "ascending": false}]}), + ) + .unwrap(); + assert_eq!(tables(), before); + } + #[test] fn test_bad_params_and_unknown_methods_are_errors() { let (mut store, reader, _) = setup(); - let session = Session { - reader: &*reader, - caps: Capabilities { regex: false }, - }; + let session = Session { reader: &*reader }; assert!(call(&mut store, &session, "result_open", &json!({})) .unwrap_err() .contains("Invalid parameters")); diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs index 5e189f6de..8b3c08d1b 100644 --- a/ggsql-jupyter/src/results/mod.rs +++ b/ggsql-jupyter/src/results/mod.rs @@ -6,23 +6,31 @@ //! calls. Nothing here knows about any particular frontend. //! //! A result is held as its Arrow data until it is first opened, and is then -//! copied into a dedicated in-memory cache ([`CacheTables`], the same +//! copied into a dedicated in-memory DuckDB ([`CacheTables`], the same //! machinery as ggsql's caching reader) where every filter, sort, page and //! profile is SQL. The cache is the results API's own: the session's reader is //! never wrapped, so nothing changes about the dialect the user's queries run //! in. +//! +//! **One byte budget covers everything kept here**: unopened results, the +//! cache's tables (including those pinned by an open view) and every view's +//! order table. Making room drops closed results first, then the oldest +//! unopened ones; an open view is never pulled from under its frontend, so +//! when open views alone fill the budget, a new result is not kept and a sort +//! or filter that needs room fails with an error saying so. mod export; pub mod filter; pub mod methods; pub mod view; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; +use filter::{Filter, SortKey}; use ggsql::naming::quote_ident; -use ggsql::reader::cache::{parse_human_bytes, CacheConfig, CacheTables}; +use ggsql::reader::cache::{estimate_bytes, CacheTables}; use ggsql::DataFrame; use serde::Serialize; -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::Arc; use view::ResultView; @@ -31,8 +39,8 @@ use view::ResultView; /// stable and clearing the sort restores the query's own order. pub const ROW: &str = "__ggsql_row__"; -/// The results cache's default byte budget. -const DEFAULT_MAX_BYTES: u64 = 1024 * 1024 * 1024; +/// The default byte budget for everything the results API keeps. +pub const DEFAULT_MAX_BYTES: u64 = 1024 * 1024 * 1024; /// A result's column. #[derive(Debug, Clone, Serialize)] @@ -43,6 +51,10 @@ pub struct Column { /// A frontend-neutral kind: `integer`, `floating`, `boolean`, `string`, /// `date`, `datetime`, `time` or `unknown`. pub kind: String, + /// Whether the query returned this name more than once, so it was renamed + /// here and SQL over the query cannot refer to it. + #[serde(skip)] + pub repeated: bool, } /// A result not yet opened, held as its Arrow data. @@ -51,6 +63,7 @@ struct Pending { title: String, query: String, df: DataFrame, + bytes: u64, } /// A result copied into the cache. @@ -77,60 +90,97 @@ pub struct OpenedView { pub struct ResultStore { /// The results cache: opened results, pinned while a view is open. cache: CacheTables, - /// Results not opened yet, oldest first, bounded by `max_pending`. + /// Results not opened yet, oldest first. pending: VecDeque, - max_pending: usize, + /// The budget for everything kept. + max_bytes: u64, opened: HashMap, views: HashMap, } impl ResultStore { - /// A store keeping up to `max_pending` unopened results. Opened results - /// are bounded by the cache's byte budget, `GGSQL_RESULTS_MAX_BYTES` - /// (default 1 GB). - pub fn new(max_pending: usize) -> Result { - let max_bytes = std::env::var("GGSQL_RESULTS_MAX_BYTES") - .ok() - .and_then(|v| parse_human_bytes(&v)) - .unwrap_or(DEFAULT_MAX_BYTES); - Self::with_max_bytes(max_pending, max_bytes) - } - - /// A store with an explicit byte budget for opened results. - pub fn with_max_bytes(max_pending: usize, max_bytes: u64) -> Result { - let config = CacheConfig { - enabled: true, - ttl_secs: u64::MAX, - max_bytes, - }; - let cache = CacheTables::in_memory(config)?; + /// A store keeping at most `max_bytes`, by Arrow's estimate of the data. + pub fn new(max_bytes: u64) -> Result { + let backend = ggsql::reader::connection::reader_from_uri("duckdb://memory")?; // An in-memory DuckDB may otherwise spill to temporary files under // memory pressure; results must never be written to disk. - cache.backend().execute_sql("SET temp_directory = ''")?; + backend.execute_sql("SET temp_directory = ''")?; + // The store evicts to its own budget, which spans more than the cache. + let cache = CacheTables::new(backend, u64::MAX); Ok(Self { cache, pending: VecDeque::new(), - max_pending: max_pending.max(1), + max_bytes, opened: HashMap::new(), views: HashMap::new(), }) } + /// Bytes kept now. + fn used(&self) -> Result { + let pending: u64 = self.pending.iter().map(|p| p.bytes).sum(); + let orders: u64 = self.views.values().map(ResultView::order_bytes).sum(); + Ok(pending + orders + self.cache.total_bytes()?) + } + + /// Drop closed results, then the oldest unopened ones, until `extra` more + /// bytes fit in the budget. Returns whether they do. + fn make_room(&mut self, extra: u64) -> Result { + let limit = self.max_bytes.saturating_sub(extra); + let outside_cache: u64 = self.pending.iter().map(|p| p.bytes).sum::() + + self + .views + .values() + .map(ResultView::order_bytes) + .sum::(); + self.cache.evict_to(limit.saturating_sub(outside_cache))?; + self.forget_evicted()?; + while self.used()? > limit && self.pending.pop_front().is_some() {} + Ok(self.used()? <= limit) + } + + /// Forget the results the cache has evicted. + fn forget_evicted(&mut self) -> Result<()> { + let mut gone = Vec::new(); + for key in self.opened.keys() { + if self.cache.lookup(key)?.is_none() { + gone.push(key.clone()); + } + } + for key in gone { + self.opened.remove(&key); + } + Ok(()) + } + /// Keep `df`, produced by `query`, under a fresh key, which is returned. - pub fn insert(&mut self, df: DataFrame, title: String, query: String) -> String { + /// `None` when it cannot fit in the budget, even once everything that can + /// go has gone. + pub fn insert( + &mut self, + df: DataFrame, + title: String, + query: String, + ) -> Result> { + let bytes = estimate_bytes(&df).max(0) as u64; + if bytes > self.max_bytes { + return Ok(None); + } // Unguessable, so a stale link from a previous kernel process can never // name a result of this one. let key = uuid::Uuid::new_v4().simple().to_string(); + let fits = self.make_room(bytes)?; + if !fits { + return Ok(None); + } self.pending.push_back(Pending { key: key.clone(), title, query, df, + bytes, }); - while self.pending.len() > self.max_pending { - self.pending.pop_front(); - } - key + Ok(Some(key)) } fn table_name(key: &str) -> String { @@ -141,28 +191,15 @@ impl ResultStore { /// first time. The result stays pinned while any view of it is open. pub fn open(&mut self, key: &str) -> Result { if let Some(i) = self.pending.iter().position(|p| p.key == key) { - let Pending { - key, - title, - query, - df, - } = self.pending.remove(i).expect("position is in range"); - match self.copy_into_cache(&key, title.clone(), query.clone(), df.clone()) { + let pending = self.pending.remove(i).expect("position is in range"); + match self.copy_into_cache(&pending) { Ok(opened) => { - self.opened.insert(key, opened); + self.opened.insert(pending.key, opened); } Err(e) => { // Kept, so the error says what went wrong each time // rather than that the result is gone. - self.pending.insert( - i, - Pending { - key, - title, - query, - df, - }, - ); + self.pending.insert(i, pending); return Err(e); } } @@ -196,21 +233,16 @@ impl ResultStore { }) } - fn copy_into_cache( - &self, - key: &str, - title: String, - query: String, - df: DataFrame, - ) -> Result { - let num_rows = df.height(); - let df = unique_names(df)?; + fn copy_into_cache(&self, pending: &Pending) -> Result { + let num_rows = pending.df.height(); + let (df, repeated) = unique_names(pending.df.clone())?; let rows = arrow::array::UInt64Array::from_iter_values(0..num_rows as u64); let df = df.with_column(ROW, Arc::new(rows))?; - let table = Self::table_name(key); + let table = Self::table_name(&pending.key); // Pinned from the start, so the eviction its own insertion triggers - // can't take it. - self.cache.put(key, &query, &table, df, true)?; + // can't take it. The budget already counts it, as a pending result. + self.cache + .put(&pending.key, &pending.query, &table, df, true)?; let described = self .cache @@ -224,11 +256,12 @@ impl ResultStore { name: names.value(i).to_string(), type_name: types.value(i).to_string(), kind: kind_of(types.value(i)).to_string(), + repeated: repeated.contains(names.value(i)), }) .collect(); Ok(Opened { - title, - query, + title: pending.title.clone(), + query: pending.query.clone(), table, columns, num_rows, @@ -237,7 +270,7 @@ impl ResultStore { /// Close a view, dropping what it keeps in the cache. Its result is /// unpinned once no view of it is open, and is then evicted when the - /// cache needs the room. + /// budget needs the room. pub fn close(&mut self, view: &str) -> Result<()> { let mut v = self .views @@ -250,6 +283,22 @@ impl ResultStore { Ok(()) } + /// Close every view, for when the frontend holding them has gone. + pub fn close_all(&mut self) { + let ids: Vec = self.views.keys().cloned().collect(); + for id in ids { + if let Err(e) = self.close(&id) { + tracing::warn!("Closing view {} failed: {}", id, e); + } + } + } + + fn view(&self, view: &str) -> Result<&ResultView> { + self.views + .get(view) + .ok_or_else(|| anyhow!("There is no view {view:?}")) + } + /// A view and the cache it reads from. pub fn view_mut(&mut self, view: &str) -> Result<(&mut ResultView, &CacheTables)> { let v = self @@ -259,24 +308,59 @@ impl ResultStore { Ok((v, &self.cache)) } + /// Make room for a new order table of `view`, which replaces its current + /// one once built. + fn reserve_order(&mut self, view: &str) -> Result<()> { + let v = self.view(view)?; + let need = v.max_order_bytes(); + if !self.make_room(need)? { + bail!( + "Sorting or filtering this result needs more memory than the data explorer's \ + budget has left ({} bytes). Close another Data Explorer tab and try again.", + self.max_bytes + ); + } + Ok(()) + } + + /// Set a view's sort keys, within the budget. + pub fn set_sort(&mut self, view: &str, keys: Vec) -> Result<(), String> { + if !keys.is_empty() { + self.reserve_order(view).map_err(|e| e.to_string())?; + } + let (v, cache) = self.view_mut(view).map_err(|e| e.to_string())?; + v.set_sort(keys, cache.backend()) + } + + /// Set a view's row filters, within the budget. + pub fn set_filters( + &mut self, + view: &str, + filters: Vec, + ) -> Result>, String> { + if !filters.is_empty() { + self.reserve_order(view).map_err(|e| e.to_string())?; + } + let (v, cache) = self.view_mut(view).map_err(|e| e.to_string())?; + v.set_filters(filters, cache.backend()) + } + /// The query a view's result came from. pub fn query_of(&self, view: &str) -> Result<&str> { - let v = self - .views - .get(view) - .ok_or_else(|| anyhow!("There is no view {view:?}"))?; + let v = self.view(view)?; Ok(&self.opened[&v.key].query) } } /// `df` with each repeated column name made unique, as DuckDB does: the second /// `a` becomes `a_1`, the third `a_2`, skipping names already taken. A query -/// can return repeated names, but a table cannot hold them. -fn unique_names(df: DataFrame) -> Result { +/// can return repeated names, but a table cannot hold them. Also returns the +/// new names. +fn unique_names(df: DataFrame) -> Result<(DataFrame, HashSet)> { let names = df.get_column_names(); - let mut taken: std::collections::HashSet = names.iter().cloned().collect(); - let mut seen = std::collections::HashSet::new(); - let mut changed = false; + let mut taken: HashSet = names.iter().cloned().collect(); + let mut seen = HashSet::new(); + let mut renamed = HashSet::new(); let mut columns = Vec::with_capacity(names.len()); for (name, array) in names.iter().zip(df.get_columns()) { let mut unique = name.clone(); @@ -287,66 +371,65 @@ fn unique_names(df: DataFrame) -> Result { } unique = format!("{name}_{n}"); taken.insert(unique.clone()); - changed = true; + renamed.insert(unique.clone()); } columns.push((unique, array.clone())); } - if !changed { - return Ok(df); + if renamed.is_empty() { + return Ok((df, renamed)); + } + // The first of a repeated name can't be named in SQL over the query either. + for name in &names { + if names.iter().filter(|n| *n == name).count() > 1 { + renamed.insert(name.clone()); + } } - Ok(DataFrame::new(columns)?) + Ok((DataFrame::new(columns)?, renamed)) } fn gone() -> anyhow::Error { anyhow!("This query result is no longer available. Re-run the query to explore it.") } -/// A cache type name's neutral kind. +/// A DuckDB type name's neutral kind. pub fn kind_of(type_name: &str) -> &'static str { - let upper = type_name.to_uppercase(); - let upper = upper.as_str(); - // Nested types first: `INTEGER[]` and `STRUCT(a INTEGER)` name a scalar - // type inside them, but are not numbers. - let nested = upper.ends_with(']') - || ["STRUCT", "MAP", "UNION", "LIST"] - .iter() - .any(|t| upper.starts_with(t)); - if nested || upper.contains("INTERVAL") { - "unknown" - } else if upper.contains("INT") { - "integer" - } else if ["FLOAT", "DOUBLE", "REAL", "NUMERIC", "DECIMAL"] - .iter() - .any(|t| upper.contains(t)) - { - "floating" - } else if upper.contains("BOOL") { - "boolean" - } else if upper.contains("TIMESTAMP") || upper.contains("DATETIME") { - "datetime" - } else if upper.contains("DATE") { - "date" - } else if upper.contains("TIME") { - "time" - } else if ["CHAR", "TEXT", "STRING", "UUID"] - .iter() - .any(|t| upper.contains(t)) - { - "string" - } else { - "unknown" + let upper = type_name.trim().to_uppercase(); + // Nested types (`INTEGER[]`, `STRUCT(a INTEGER)`) name a scalar type + // inside them, but are not one. + if upper.ends_with(']') { + return "unknown"; + } + // The type itself, without parameters: `DECIMAL(18,3)` is `DECIMAL`, + // `ENUM('a', 'b')` is `ENUM`. + let base = upper.split('(').next().unwrap_or("").trim(); + match base { + "TINYINT" | "SMALLINT" | "INTEGER" | "BIGINT" | "HUGEINT" | "UTINYINT" | "USMALLINT" + | "UINTEGER" | "UBIGINT" | "UHUGEINT" | "INT" | "INT1" | "INT2" | "INT4" | "INT8" => { + "integer" + } + "FLOAT" | "REAL" | "DOUBLE" | "DECIMAL" | "NUMERIC" => "floating", + "BOOLEAN" | "BOOL" => "boolean", + "DATE" => "date", + "TIMESTAMP" + | "TIMESTAMP WITH TIME ZONE" + | "TIMESTAMPTZ" + | "TIMESTAMP_S" + | "TIMESTAMP_MS" + | "TIMESTAMP_NS" + | "DATETIME" => "datetime", + "TIME" | "TIME WITH TIME ZONE" | "TIMETZ" => "time", + "VARCHAR" | "CHAR" | "BPCHAR" | "TEXT" | "STRING" | "UUID" | "ENUM" => "string", + _ => "unknown", } } #[cfg(test)] mod tests { use super::*; - use crate::results::filter::{Capabilities, Filter, SortKey}; + use crate::results::filter::{Filter, SortKey}; use crate::results::view::Rows; use serde_json::json; - const CAPS: Capabilities = Capabilities { regex: true }; - fn frame(sql: &str) -> DataFrame { ggsql::reader::connection::reader_from_uri("duckdb://memory") .unwrap() @@ -354,6 +437,13 @@ mod tests { .unwrap() } + fn keep(store: &mut ResultStore, df: DataFrame, title: String, query: String) -> String { + store + .insert(df, title, query) + .unwrap() + .expect("the result fits") + } + fn numbers(n: usize) -> DataFrame { frame(&format!( "SELECT i AS n, i % 3 AS g, 'item ' || i AS s FROM range({n}) t(i)" @@ -378,8 +468,13 @@ mod tests { #[test] fn test_open_describes_the_result_without_the_row_column() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(numbers(5), "Query [1]".into(), "SELECT 1".into()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut store, + numbers(5), + "Query [1]".into(), + "SELECT 1".into(), + ); let opened = store.open(&key).unwrap(); assert_eq!(opened.num_rows, 5); let names: Vec<_> = opened.columns.iter().map(|c| c.name.as_str()).collect(); @@ -391,8 +486,8 @@ mod tests { #[test] fn test_sort_is_stable_and_clearing_it_restores_the_order() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(numbers(9), "q".into(), String::new()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(9), "q".into(), String::new()); let view = store.open(&key).unwrap().view; let (v, cache) = store.view_mut(&view).unwrap(); v.set_sort( @@ -414,8 +509,8 @@ mod tests { #[test] fn test_filters_select_rows_and_report_bad_ones() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(numbers(10), "q".into(), String::new()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(10), "q".into(), String::new()); let view = store.open(&key).unwrap().view; let filters: Vec = serde_json::from_value(json!([ {"type": "compare", "column": 0, "op": ">=", "value": "4"}, @@ -424,7 +519,7 @@ mod tests { ])) .unwrap(); let (v, cache) = store.view_mut(&view).unwrap(); - let errors = v.set_filters(filters, cache.backend(), CAPS).unwrap(); + let errors = v.set_filters(filters, cache.backend()).unwrap(); assert!(errors[0].is_none()); assert!(errors[1].as_deref().unwrap().contains("not a number")); assert!(errors[2].is_none()); @@ -434,8 +529,9 @@ mod tests { #[test] fn test_dates_page_and_profile_as_dates() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert( + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut store, frame("SELECT DATE '2024-01-01' + i::INTEGER AS d FROM range(3) t(i)"), "q".into(), String::new(), @@ -453,8 +549,8 @@ mod tests { } fn summary_of(sql: &str, column: usize) -> serde_json::Value { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(frame(sql), "q".into(), String::new()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, frame(sql), "q".into(), String::new()); let view = store.open(&key).unwrap().view; let (v, cache) = store.view_mut(&view).unwrap(); let req: view::ProfileRequest = @@ -492,8 +588,9 @@ mod tests { #[test] fn test_repeated_column_names_are_made_unique() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert( + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut store, frame("SELECT 1 AS a, 2 AS a, 3 AS a_1"), "q".into(), String::new(), @@ -506,10 +603,15 @@ mod tests { #[test] fn test_a_failed_open_keeps_the_result() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(frame("SELECT 1 AS x"), "q".into(), String::new()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut store, + frame("SELECT 1 AS x"), + "q".into(), + String::new(), + ); // Break the cache's bookkeeping, so copying the result in fails. - store.cache.ensure_meta_table().unwrap(); + store.cache.lookup("anything").unwrap(); store .cache .backend() @@ -527,8 +629,9 @@ mod tests { #[test] fn test_sql_leaves_out_a_filter_the_cache_rejected() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert( + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut store, frame("SELECT 'a' AS s FROM range(3)"), "q".into(), "SELECT 1".into(), @@ -541,9 +644,9 @@ mod tests { ])) .unwrap(); let (v, cache) = store.view_mut(&view).unwrap(); - let errors = v.set_filters(filters, cache.backend(), CAPS).unwrap(); + let errors = v.set_filters(filters, cache.backend()).unwrap(); assert!(errors[0].is_some() && errors[1].is_none(), "{errors:?}"); - let sql = v.to_sql("SELECT 1", &ggsql::reader::AnsiDialect, CAPS); + let sql = v.to_sql("SELECT 1", &ggsql::reader::AnsiDialect); assert!(sql[0].starts_with("-- Filter left out"), "{sql:?}"); assert!(!sql.iter().any(|l| l.contains("regexp_matches")), "{sql:?}"); assert!(sql.iter().any(|l| l.contains("IS NOT NULL")), "{sql:?}"); @@ -551,16 +654,17 @@ mod tests { #[test] fn test_paging_past_the_end_returns_what_there_is() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(numbers(3), "q".into(), String::new()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(3), "q".into(), String::new()); let view = store.open(&key).unwrap().view; assert_eq!(page(&mut store, &view, 2, 10, 0), ["2"]); } #[test] fn test_profiles() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert( + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut store, frame("SELECT * FROM (VALUES (1, 'a'), (2, 'b'), (3, 'a'), (NULL, NULL)) t(n, s)"), "q".into(), String::new(), @@ -595,8 +699,8 @@ mod tests { #[test] fn test_export_follows_the_view() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(numbers(4), "q".into(), String::new()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(4), "q".into(), String::new()); let view = store.open(&key).unwrap().view; let (v, cache) = store.view_mut(&view).unwrap(); v.set_sort( @@ -618,26 +722,215 @@ mod tests { assert_eq!(csv, "n,s\n3,item 3\n2,item 2"); } + fn bytes_of(df: &DataFrame) -> u64 { + estimate_bytes(df) as u64 + } + #[test] fn test_an_open_result_survives_the_budget_and_a_closed_one_does_not() { - let mut store = ResultStore::with_max_bytes(4, 1).unwrap(); - let a = store.insert(numbers(3), "a".into(), String::new()); - let b = store.insert(numbers(3), "b".into(), String::new()); + // Room for two results and an order table, not three results. + let one = bytes_of(&numbers(100)); + let mut store = ResultStore::new(2 * one + 100 * 16).unwrap(); + let a = keep(&mut store, numbers(100), "a".into(), String::new()); + let b = keep(&mut store, numbers(100), "b".into(), String::new()); let va = store.open(&a).unwrap().view; let vb = store.open(&b).unwrap().view; - // Both open: both pinned, whatever the budget. + + // Both open: a third result does not fit, and nothing is dropped. + assert!(store + .insert(numbers(100), "c".into(), String::new()) + .unwrap() + .is_none()); assert_eq!(page(&mut store, &va, 0, 0, 0), ["0"]); assert_eq!(page(&mut store, &vb, 0, 0, 0), ["0"]); + // Closed, `a` gives way to a new result. store.close(&va).unwrap(); + let c = store + .insert(numbers(100), "c".into(), String::new()) + .unwrap(); + assert!(c.is_some()); let err = store.open(&a).unwrap_err().to_string(); assert!(err.contains("no longer available"), "{err}"); assert!(store.open(&b).is_ok()); } + #[test] + fn test_the_oldest_unopened_result_is_dropped_to_make_room() { + let one = bytes_of(&numbers(10)); + let mut store = ResultStore::new(2 * one).unwrap(); + let first = keep(&mut store, numbers(10), "a".into(), String::new()); + let second = keep(&mut store, numbers(10), "b".into(), String::new()); + keep(&mut store, numbers(10), "c".into(), String::new()); + assert!(store.open(&first).is_err()); + assert!(store.open(&second).is_ok()); + } + + #[test] + fn test_a_result_bigger_than_the_budget_is_not_kept() { + let mut store = ResultStore::new(1).unwrap(); + assert!(store + .insert(numbers(10), "a".into(), String::new()) + .unwrap() + .is_none()); + } + + #[test] + fn test_order_tables_count_against_the_budget() { + let one = bytes_of(&numbers(1000)); + // Room for the result and less than one order table. + let mut store = ResultStore::new(one + 1000).unwrap(); + let key = keep(&mut store, numbers(1000), "a".into(), String::new()); + let view = store.open(&key).unwrap().view; + let err = store + .set_sort( + &view, + vec![SortKey { + column: 0, + ascending: false, + }], + ) + .unwrap_err(); + assert!(err.contains("memory"), "{err}"); + // The view is untouched. + assert_eq!(page(&mut store, &view, 0, 1, 0), ["0", "1"]); + } + + #[test] + fn test_close_all_unpins_every_result() { + let one = bytes_of(&numbers(100)); + let mut store = ResultStore::new(one).unwrap(); + let a = keep(&mut store, numbers(100), "a".into(), String::new()); + store.open(&a).unwrap(); + store.open(&a).unwrap(); + store.close_all(); + assert!(store.views.is_empty()); + // Unpinned, it gives way to the next result. + assert!(store + .insert(numbers(100), "b".into(), String::new()) + .unwrap() + .is_some()); + assert!(store.open(&a).is_err()); + assert!(store.opened.is_empty() || !store.opened.contains_key(&a)); + } + + #[test] + fn test_a_failed_sort_leaves_the_view_as_it_was() { + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(9), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + store + .set_sort( + &view, + vec![SortKey { + column: 1, + ascending: false, + }], + ) + .unwrap(); + let before = page(&mut store, &view, 0, 8, 0); + // A key the database cannot sort on. + let (v, cache) = store.view_mut(&view).unwrap(); + v.columns[0].name = "no such column".into(); + assert!(v + .set_sort( + vec![SortKey { + column: 0, + ascending: true + }], + cache.backend() + ) + .is_err()); + assert_eq!( + v.sort, + vec![SortKey { + column: 1, + ascending: false + }] + ); + v.columns[0].name = "n".into(); + assert_eq!(page(&mut store, &view, 0, 8, 0), before); + } + + #[test] + fn test_a_failed_filter_leaves_the_view_as_it_was() { + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(10), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + let gt = |v: &str| -> Vec { + serde_json::from_value(json!([{"type": "compare", "column": 0, "op": ">", "value": v}])) + .unwrap() + }; + store.set_filters(&view, gt("6")).unwrap(); + // Make the next rebuild fail: its order table's name is taken by a view. + store + .cache + .backend() + .execute_sql(&format!( + "CREATE TEMP VIEW {} AS SELECT 1 AS x", + quote_ident(&format!("__ggsql_view_{view}_1__")) + )) + .unwrap(); + assert!(store.set_filters(&view, gt("1")).is_err()); + let (v, _) = store.view_mut(&view).unwrap(); + assert_eq!(v.selected_rows, 3); + assert_eq!(v.filters, gt("6")); + assert_eq!(page(&mut store, &view, 0, 9, 0), ["7", "8", "9"]); + } + + #[test] + fn test_pages_are_bounded_and_never_shift_rows() { + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(20), "q".into(), String::new()); + let view = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&view).unwrap(); + let get = |rows: Rows| v.page(cache.backend(), &rows, &[0]); + + // Scattered rows come back in the order asked, without the rows between. + let cells = get(Rows::Indices { + indices: vec![15, 2, 15], + }) + .unwrap(); + assert_eq!(cells[0], vec![json!("15"), json!("2"), json!("15")]); + // A row outside the view is an error, not a short column. + assert!(get(Rows::Indices { + indices: vec![3, 20] + }) + .unwrap_err() + .contains("past the end")); + // A huge range is refused, not materialized. + assert!(get(Rows::Range { + first: 0, + last: usize::MAX + }) + .unwrap() + .first() + .is_some_and(|c| c.len() == 20)); + let mut big = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut big, + numbers(view::MAX_PAGE_ROWS + 1), + "q".into(), + String::new(), + ); + let bv = big.open(&key).unwrap().view; + let (v, cache) = big.view_mut(&bv).unwrap(); + assert!(v + .page( + cache.backend(), + &Rows::Range { + first: 0, + last: usize::MAX + }, + &[0] + ) + .unwrap_err() + .contains("at most")); + } + #[test] fn test_running_out_of_memory_is_an_error_not_a_spill() { - let mut store = ResultStore::new(4).unwrap(); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); let setting = store .cache .backend() @@ -650,7 +943,8 @@ mod tests { "" ); - let key = store.insert( + let key = keep( + &mut store, frame("SELECT i AS n, md5(i::VARCHAR) AS s FROM range(200000) t(i)"), "q".into(), String::new(), @@ -691,8 +985,8 @@ mod tests { #[test] fn test_reopening_reuses_the_cached_copy() { - let mut store = ResultStore::new(4).unwrap(); - let key = store.insert(numbers(3), "q".into(), String::new()); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(3), "q".into(), String::new()); let first = store.open(&key).unwrap().view; store.close(&first).unwrap(); let second = store.open(&key).unwrap(); @@ -700,19 +994,9 @@ mod tests { assert_eq!(second.num_rows, 3); } - #[test] - fn test_the_oldest_unopened_result_is_dropped_past_the_limit() { - let mut store = ResultStore::new(2).unwrap(); - let first = store.insert(numbers(1), "a".into(), String::new()); - let second = store.insert(numbers(1), "b".into(), String::new()); - store.insert(numbers(1), "c".into(), String::new()); - assert!(store.open(&first).is_err()); - assert!(store.open(&second).is_ok()); - } - #[test] fn test_an_unknown_key_says_the_result_is_gone() { - let mut store = ResultStore::new(2).unwrap(); + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); let err = store.open("nope").err().unwrap().to_string(); assert!(err.contains("no longer available"), "{err}"); } @@ -725,5 +1009,12 @@ mod tests { assert_eq!(kind_of("DATE"), "date"); assert_eq!(kind_of("VARCHAR"), "string"); assert_eq!(kind_of("INTERVAL"), "unknown"); + // Names that merely contain a type's name are not that type. + assert_eq!(kind_of("POINT"), "unknown"); + assert_eq!(kind_of("GEOMETRY"), "unknown"); + assert_eq!(kind_of("ENUM('PRINT', 'POINT')"), "string"); + assert_eq!(kind_of("TIMESTAMP_NS"), "datetime"); + assert_eq!(kind_of("TIME WITH TIME ZONE"), "time"); + assert_eq!(kind_of("BIGINT[]"), "unknown"); } } diff --git a/ggsql-jupyter/src/results/view.rs b/ggsql-jupyter/src/results/view.rs index 0c4019681..26de3a871 100644 --- a/ggsql-jupyter/src/results/view.rs +++ b/ggsql-jupyter/src/results/view.rs @@ -6,8 +6,11 @@ //! position in the view. A page then joins that small table back to the //! result, so scrolling never sorts again. An unfiltered, unsorted view needs //! no order table: its positions are the `ROW` values themselves. +//! +//! A new sort or set of filters builds its order table under a new name and +//! only then replaces the old one, so one that fails leaves the view as it was. -use super::filter::{self, Capabilities, Filter, SortKey}; +use super::filter::{self, Filter, SortKey}; use super::{export, Column, ROW}; use ggsql::naming::quote_ident; use ggsql::reader::Reader; @@ -15,24 +18,29 @@ use ggsql::sql::FromItem; use ggsql::DataFrame; use serde::Deserialize; use serde_json::{json, Value}; +use std::collections::HashMap; + +/// The most rows one page may ask for. The Data Explorer asks for a screenful. +pub const MAX_PAGE_ROWS: usize = 10_000; + +/// Bytes an order table holds per row: two 64-bit integers. +pub const ORDER_ROW_BYTES: u64 = 16; /// The rows of a view to read. #[derive(Debug, Clone, Deserialize)] #[serde(untagged)] pub enum Rows { - /// `first..=last`. + /// `first..=last`. Rows past the end of the view are not returned. Range { first: usize, last: usize }, - /// These positions, in this order. + /// These positions, in this order. Each must be in the view. Indices { indices: Vec }, } -impl Rows { - fn positions(&self) -> Vec { - match self { - Rows::Range { first, last } => (*first..=*last).collect(), - Rows::Indices { indices } => indices.clone(), - } - } +/// Which of a view's rows to fetch. +enum Fetch<'a> { + All, + Range(usize, usize), + Positions(&'a [usize]), } /// Which profiles to compute for a column. @@ -72,12 +80,14 @@ fn default_limit() -> usize { pub struct ResultView { /// The result this views. pub key: String, + /// The view's id, which its order tables are named after. + id: String, /// The result table, quoted. table: String, - /// The order table's name, used once the view is filtered or sorted. - order_name: String, - /// Whether the order table exists. - ordered: bool, + /// The order table, once the view is filtered or sorted. + order: Option, + /// How many order tables this view has built, for the next one's name. + built: usize, pub columns: Vec, /// Rows in the result. pub num_rows: usize, @@ -96,9 +106,10 @@ impl ResultView { pub fn new(key: &str, id: &str, table: &str, columns: Vec, num_rows: usize) -> Self { Self { key: key.to_string(), + id: id.to_string(), table: quote_ident(table), - order_name: format!("__ggsql_view_{id}__"), - ordered: false, + order: None, + built: 0, columns, num_rows, selected_rows: num_rows, @@ -109,6 +120,20 @@ impl ResultView { } } + /// The bytes the view's order table holds. + pub fn order_bytes(&self) -> u64 { + if self.order.is_some() { + self.selected_rows as u64 * ORDER_ROW_BYTES + } else { + 0 + } + } + + /// The most bytes an order table of this view can hold. + pub fn max_order_bytes(&self) -> u64 { + self.num_rows as u64 * ORDER_ROW_BYTES + } + /// The result table filtered, as a `FROM` item, for the queries that /// aggregate it. fn relation(&self) -> String { @@ -123,17 +148,17 @@ impl ResultView { /// Set the row filters, returning an error message per filter (`None` for /// a valid one). A filter that cannot be built, or that the database - /// rejects, is left out rather than failing the rest. + /// rejects, is left out rather than failing the rest. If the view cannot + /// be rebuilt with them, it is left as it was and the error returned. pub fn set_filters( &mut self, filters: Vec, reader: &dyn Reader, - caps: Capabilities, ) -> Result>, String> { let mut errors: Vec> = vec![None; filters.len()]; let mut built = Vec::new(); for (i, f) in filters.iter().enumerate() { - match filter::predicate(f, &self.columns, reader.dialect(), caps) { + match filter::predicate(f, &self.columns, reader.dialect()) { Ok(p) => built.push((i, p)), Err(e) => errors[i] = Some(e), } @@ -159,35 +184,44 @@ impl ResultView { }); } + let predicate = combine(&built); + self.rebuild(reader, predicate.clone(), &self.sort.clone())?; self.filters = filters; self.rejected = errors.clone(); - self.predicate = combine(&built); - self.rebuild(reader)?; + self.predicate = predicate; Ok(errors) } - /// Set the sort keys. + /// Set the sort keys. If the view cannot be rebuilt with them, it is left + /// as it was and the error returned. pub fn set_sort(&mut self, keys: Vec, reader: &dyn Reader) -> Result<(), String> { - filter::order_terms(&keys, &self.columns)?; + filter::order_terms(&keys, &self.columns, reader.dialect())?; + self.rebuild(reader, self.predicate.clone(), &keys)?; self.sort = keys; - self.rebuild(reader) + Ok(()) } - /// Rebuild the order table for the current filters and sort, and count - /// the rows they select. - fn rebuild(&mut self, reader: &dyn Reader) -> Result<(), String> { - self.drop_order(reader); - if self.predicate.is_none() && self.sort.is_empty() { + /// Build the order table for `predicate` and `sort`, then replace the + /// view's with it and count the rows it selects. On an error the view is + /// unchanged. + fn rebuild( + &mut self, + reader: &dyn Reader, + predicate: Option, + sort: &[SortKey], + ) -> Result<(), String> { + if predicate.is_none() && sort.is_empty() { + self.drop_order(reader); self.selected_rows = self.num_rows; return Ok(()); } // Ties break on the query's own order, so paging is stable and clearing // the sort restores it. - let mut terms = filter::order_terms(&self.sort, &self.columns)?; + let dialect = reader.dialect(); + let mut terms = filter::order_terms(sort, &self.columns, dialect)?; terms.push(quote_ident(ROW)); - let filter = self - .predicate + let filter = predicate .as_ref() .map(|p| format!(" WHERE {p}")) .unwrap_or_default(); @@ -197,22 +231,33 @@ impl ResultView { self.table, row = quote_ident(ROW), ); - let dialect = reader.dialect(); - for stmt in dialect.create_or_replace_temp_table_sql(&self.order_name, &[], &body) { - reader.execute_sql(&stmt).map_err(|e| e.to_string())?; - } - self.ordered = true; - self.selected_rows = count_rows(reader, "e_ident(&self.order_name), None)?; + let name = format!("__ggsql_view_{}_{}__", self.id, self.built); + let built = (|| { + for stmt in dialect.create_or_replace_temp_table_sql(&name, &[], &body) { + reader.execute_sql(&stmt).map_err(|e| e.to_string())?; + } + count_rows(reader, "e_ident(&name), None) + })(); + let selected = match built { + Ok(n) => n, + Err(e) => { + let _ = reader.execute_sql(&dialect.drop_table_sql(&name)); + return Err(e); + } + }; + self.drop_order(reader); + self.built += 1; + self.order = Some(name); + self.selected_rows = selected; Ok(()) } fn drop_order(&mut self, reader: &dyn Reader) { - if self.ordered { - let sql = reader.dialect().drop_table_sql(&self.order_name); + if let Some(name) = self.order.take() { + let sql = reader.dialect().drop_table_sql(&name); if let Err(e) = reader.execute_sql(&sql) { - tracing::warn!("Failed to drop {}: {}", self.order_name, e); + tracing::warn!("Failed to drop {}: {}", name, e); } - self.ordered = false; } } @@ -221,12 +266,13 @@ impl ResultView { self.drop_order(reader); } - /// Read the view's rows `first..=last`, or every row, for `cols`. + /// Read `which` of the view's rows for `cols`, in the view's order, each + /// with its position as the frame's first column. fn fetch( &self, reader: &dyn Reader, cols: &[usize], - span: Option<(usize, usize)>, + which: Fetch, ) -> Result { let names = cols .iter() @@ -237,33 +283,97 @@ impl ResultView { .ok_or_else(|| format!("There is no column {i}")) }) .collect::, _>>()?; - if names.is_empty() { - return Ok(DataFrame::empty()); - } let row = quote_ident(ROW); - let sql = if self.ordered { - let range = span - .map(|(a, b)| format!(" WHERE o.\"pos\" BETWEEN {a} AND {b}")) - .unwrap_or_default(); - format!( - "SELECT {} FROM {} AS o JOIN {} AS t ON t.{row} = o.{row}{range} ORDER BY o.\"pos\"", - names.join(", "), - quote_ident(&self.order_name), - self.table, - ) - } else { - let range = span - .map(|(a, b)| format!(" WHERE t.{row} BETWEEN {a} AND {b}")) - .unwrap_or_default(); - format!( - "SELECT {} FROM {} AS t{range} ORDER BY t.{row}", - names.join(", "), - self.table, - ) + let (pos, from) = match &self.order { + Some(order) => ( + "o.\"pos\"".to_string(), + format!( + "{} AS o JOIN {} AS t ON t.{row} = o.{row}", + quote_ident(order), + self.table + ), + ), + None => (format!("t.{row}"), format!("{} AS t", self.table)), + }; + let filter = match which { + Fetch::All => String::new(), + Fetch::Range(a, b) => format!(" WHERE {pos} BETWEEN {a} AND {b}"), + Fetch::Positions(list) => { + let list: Vec = list.iter().map(usize::to_string).collect(); + format!(" WHERE {pos} IN ({})", list.join(", ")) + } }; + let select: Vec = std::iter::once(format!("{pos} AS \"__ggsql_pos__\"")) + .chain(names) + .collect(); + let sql = format!( + "SELECT {} FROM {from}{filter} ORDER BY {pos}", + select.join(", ") + ); reader.execute_sql(&sql).map_err(|e| e.to_string()) } + /// The frame rows for `rows`, in order: a range is cut short at the end of + /// the view, and a position outside it is an error rather than a row that + /// lands in another's place. + fn fetch_rows( + &self, + reader: &dyn Reader, + cols: &[usize], + rows: &Rows, + ) -> Result<(DataFrame, Vec), String> { + let n = self.selected_rows; + match rows { + Rows::Range { first, last } => { + let last = (*last).min(n.saturating_sub(1)); + if n == 0 || first > &last { + return Ok((DataFrame::empty(), Vec::new())); + } + if last - first + 1 > MAX_PAGE_ROWS { + return Err(too_many(last - first + 1)); + } + let df = self.fetch(reader, cols, Fetch::Range(*first, last))?; + let picked = (0..df.height()).collect(); + Ok((df, picked)) + } + Rows::Indices { indices } => { + if indices.len() > MAX_PAGE_ROWS { + return Err(too_many(indices.len())); + } + if let Some(bad) = indices.iter().find(|&&i| i >= n) { + return Err(format!( + "Row {bad} is past the end of the view, which has {n}" + )); + } + if indices.is_empty() { + return Ok((DataFrame::empty(), Vec::new())); + } + let mut unique = indices.clone(); + unique.sort_unstable(); + unique.dedup(); + let df = self.fetch(reader, cols, Fetch::Positions(&unique))?; + let pos = df.column("__ggsql_pos__").map_err(|e| e.to_string())?; + let at: HashMap = (0..df.height()) + .filter_map(|r| { + ggsql::array_util::value_to_string(pos, r) + .parse::() + .ok() + .map(|p| (p, r)) + }) + .collect(); + let picked = indices + .iter() + .map(|i| { + at.get(i) + .copied() + .ok_or_else(|| format!("Row {i} was not found")) + }) + .collect::, _>>()?; + Ok((df, picked)) + } + } + } + /// The cells at `rows` of each of `cols`, as `columns[col][row]`: a string, /// or `null` for a null. pub fn page( @@ -272,22 +382,14 @@ impl ResultView { rows: &Rows, cols: &[usize], ) -> Result>, String> { - let positions = rows.positions(); - let (Some(&first), Some(&last)) = (positions.iter().min(), positions.iter().max()) else { + let (df, picked) = self.fetch_rows(reader, cols, rows)?; + if picked.is_empty() { return Ok(vec![Vec::new(); cols.len()]); - }; - let df = self.fetch(reader, cols, Some((first, last)))?; - Ok(df - .get_columns() + } + // The first column is each row's position. + Ok(df.get_columns()[1..] .iter() - .map(|col| { - positions - .iter() - .map(|&r| r - first) - .filter(|&r| r < df.height()) - .map(|r| cell_value(col, r)) - .collect() - }) + .map(|col| picked.iter().map(|&r| cell_value(col, r)).collect()) .collect()) } @@ -300,41 +402,29 @@ impl ResultView { cols: &[usize], format: &str, ) -> Result { - let (df, picked): (DataFrame, Vec) = match rows { + let (df, picked) = match rows { None => { - let df = self.fetch(reader, cols, None)?; + let df = self.fetch(reader, cols, Fetch::All)?; let n = df.height(); (df, (0..n).collect()) } - Some(rows) => { - let positions = rows.positions(); - let first = positions.iter().min().copied().unwrap_or(0); - let last = positions.iter().max().copied().unwrap_or(0); - let df = self.fetch(reader, cols, Some((first, last)))?; - let n = df.height(); - let picked = positions - .iter() - .map(|r| r - first) - .filter(|&r| r < n) - .collect(); - (df, picked) - } + Some(rows) => self.fetch_rows(reader, cols, rows)?, }; let cells: Vec> = picked .iter() - .map(|&r| df.get_columns().iter().map(|c| cell_string(c, r)).collect()) + .map(|&r| { + df.get_columns()[1..] + .iter() + .map(|c| cell_string(c, r)) + .collect() + }) .collect(); let header: Vec = cols.iter().map(|&i| self.columns[i].name.clone()).collect(); export::format(Some(&header), &cells, format) } /// SQL for `dialect` reproducing this view over the result's `query`. - pub fn to_sql( - &self, - query: &str, - dialect: &dyn ggsql::reader::SqlDialect, - caps: Capabilities, - ) -> Vec { + pub fn to_sql(&self, query: &str, dialect: &dyn ggsql::reader::SqlDialect) -> Vec { // Only the filters the view applies: one the cache rejected would make // the SQL fail, or select rows the view does not show. let mut applied = Vec::new(); @@ -348,10 +438,9 @@ impl ResultView { lines.extend(filter::to_sql( &applied, &self.sort, - &filter::subquery(query), + query, &self.columns, dialect, - caps, )); lines } @@ -628,6 +717,10 @@ impl ResultView { } } +fn too_many(n: usize) -> String { + format!("A page can hold at most {MAX_PAGE_ROWS} rows, not {n}") +} + /// `SELECT COUNT(*)` over `table`, filtered by `predicate` when given. pub fn count_rows( reader: &dyn Reader, diff --git a/ggsql-jupyter/tests/test_integration.py b/ggsql-jupyter/tests/test_integration.py index 83fee6e66..b8dd9ccb0 100644 --- a/ggsql-jupyter/tests/test_integration.py +++ b/ggsql-jupyter/tests/test_integration.py @@ -109,9 +109,12 @@ def console_kernel_manager(kernel_binary): is authoritative over the session-id heuristic — so this is the only way to reach the plot-comm path from a test. Everything driven through plain `jupyter_client` lands in `SessionKind::Standalone`, because its session id - is a bare UUID. + is a bare UUID. `GGSQL_KEEP_RESULTS` is what the extension sets for a console + whose results it can open in the Data Explorer. """ - yield from _launch(kernel_binary, ["--session-mode", "console"]) + yield from _launch( + kernel_binary, ["--session-mode", "console"], env={"GGSQL_KEEP_RESULTS": "1"} + ) @pytest.fixture @@ -620,13 +623,95 @@ def test_an_unknown_call_method_is_an_error(self, console_client): finally: self._close_ui(console_client) - def test_only_a_console_keeps_results(self, client): + def test_results_are_kept_only_when_asked(self, client): client.execute(self.QUERY, silent=False, store_history=True) messages = drain_iopub(client) client.get_shell_msg(timeout=5) result = next(m for m in messages if m["msg_type"] == "execute_result") assert "ggsql" not in result["content"]["metadata"] + def _scalar(self, client, sql): + """The single value `sql` returns, as text.""" + client.execute(sql, silent=False, store_history=True) + messages = drain_iopub(client) + client.get_shell_msg(timeout=5) + result = next(m for m in messages if m["msg_type"] == "execute_result") + return result["content"]["data"]["text/plain"].splitlines()[-1].strip() + + def test_profile_and_export(self, console_client): + key = self._result_key(console_client) + self._open_ui(console_client) + try: + view = self._call(console_client, "result_open", {"key": key})["result"]["view"] + profile = self._call( + console_client, + "result_profile", + { + "view": view, + "column": 0, + "profiles": { + "null_count": True, + "summary": True, + "histogram": {"bins": 5, "quantiles": [0.5]}, + }, + }, + )["result"] + assert profile["null_count"] == 0 + assert profile["summary"]["min"] == "0" + assert profile["summary"]["max"] == "4" + assert sum(profile["histogram"]["bin_counts"]) == 5 + + frequency = self._call( + console_client, + "result_profile", + {"view": view, "column": 1, "profiles": {"frequency": {"limit": 2}}}, + )["result"]["frequency"] + assert len(frequency["values"]) == 2 + assert frequency["other_count"] == 3 + + export = self._call( + console_client, + "result_export", + {"view": view, "rows": {"first": 3, "last": 4}, "columns": [1, 0], "format": "csv"}, + )["result"] + assert export == {"data": "label,n\nr3,3\nr4,4"} + finally: + self._close_ui(console_client) + + def test_the_session_database_is_untouched(self, console_client): + count = ( + "SELECT count(*) AS n FROM duckdb_tables() " + "WHERE table_name LIKE '__ggsql_result%' OR table_name LIKE '__ggsql_view%'" + ) + key = self._result_key(console_client) + self._open_ui(console_client) + try: + view = self._call(console_client, "result_open", {"key": key})["result"]["view"] + self._call( + console_client, + "result_set_sort", + {"view": view, "keys": [{"column": 0, "ascending": False}]}, + ) + assert self._scalar(console_client, count) == "0" + finally: + self._close_ui(console_client) + + def test_closing_the_ui_comm_closes_its_views(self, console_client): + key = self._result_key(console_client) + self._open_ui(console_client) + view = self._call(console_client, "result_open", {"key": key})["result"]["view"] + self._close_ui(console_client) + self._open_ui(console_client) + try: + data = self._call( + console_client, + "result_page", + {"view": view, "rows": {"first": 0, "last": 0}, "columns": [0]}, + ) + assert "no view" in data["error"]["message"], data + finally: + self._close_ui(console_client) + class TestConsoleRows: """`GGSQL_CONSOLE_ROWS`, which the extension sets from `ggsql.consoleRows`.""" diff --git a/ggsql-vscode/AGENTS.md b/ggsql-vscode/AGENTS.md index e18bbb40f..f569ca6b1 100644 --- a/ggsql-vscode/AGENTS.md +++ b/ggsql-vscode/AGENTS.md @@ -101,7 +101,9 @@ The extension declares `contributes.languageRuntimes` for `ggsql` (see `package. - **`createKernelSpec` appends `--session-mode `** from `sessionMetadata.sessionMode`. The kernel keys every output decision off that: a console session gets the comm, a notebook session gets a static image in its cell, and background gets the standalone path. Without the flag the kernel falls back to guessing from its session id, which works but is a heuristic. `restoreSession` deliberately does *not* rebuild the spec — the supervisor replays the argv the session was created with, and a session's mode never changes. - **`uiSubscriptions: ['did_change_plots_render_settings']`** on the runtime metadata. Without it the frontend never tells the kernel how large the Plots pane is, and the kernel needs that for exactly one thing: rendering a *new* plot at the right size so its `comm_open` can carry it, and the pane shows it immediately instead of blank. Every other render carries its own size on the request. -Both live in `manager.ts`, as does one piece of user configuration: **`ggsql.consoleRows` reaches the kernel as `GGSQL_CONSOLE_ROWS` in the spec's `env`, not as `--console-rows`.** A kernel found on the machine may predate the option, and it ignores an unknown variable where it would refuse to start on an unknown flag. `consoleRowsEnv` leaves out a value the kernel would reject, for the same reason. Like the session mode, it is fixed when a session starts. +Both live in `manager.ts`, as does one piece of user configuration: **`ggsql.consoleRows` reaches the kernel as `GGSQL_CONSOLE_ROWS` in the spec's `env`, not as `--console-rows`.** A kernel found on the machine may predate the option, and it ignores an unknown variable where it would refuse to start on an unknown flag. `consoleRowsEnv` leaves out a value the kernel would reject — below 1, or above `u32::MAX`, which the setting's `maximum` also states — for the same reason. Like the session mode, it is fixed when a session starts. + +The same goes for **`GGSQL_KEEP_RESULTS`**, which `keepResultsEnv` sets for a console session when this Positron has extension-hosted Data Explorer backends: the kernel keeps query results only when asked, so a session whose results nothing can open holds none. The kernel side of the contract — why the comm alone creates the pane entry, and why an output bundle alongside it would show the plot twice — is in [`/ggsql-jupyter/AGENTS.md`](../ggsql-jupyter/AGENTS.md). @@ -129,7 +131,9 @@ A console query result opens in Positron's Data Explorer, served by a backend th **The backend** (`src/dataExplorer/`) registers with `positron.dataExplorer.registerRpcHandler('ggsql', …)` from `extension.ts`, before the runtime manager, which is handed it as `resultExplorer`. `package.json` carries the `onPositronDataExplorerBackend:ggsql` activation event, which Positron fires before routing a request to a provider that has not registered yet — for example a Data Explorer restored by a window reload. -- A dataset id is `ggsql:/`. The scheme is required: Positron puts the id in the explorer editor's URI and parses it back out (`PositronDataExplorerUri.parse`), accepting only a UUID or `:`, and an id it cannot parse opens an empty editor. `ResultExplorer` keeps a registry of the sessions the manager wrapped, and each open dataset holds one kernel view: opened with `result_open` on the first request or when a link is clicked, closed with `result_close` from `closeDataset`. The views of a session are forgotten when its kernel exits or restarts, since they ended with it, and each open explorer is sent `schema_update`, so it asks for its state again: the result reopens, or the explorer shows the kernel's reason it is gone. A `get_state` for a result that cannot be opened answers a disconnected state carrying the kernel's message, which the Data Explorer shows in place of the table. +- A dataset id is `ggsql:/`. The scheme is required: Positron puts the id in the explorer editor's URI and parses it back out (`PositronDataExplorerUri.parse`), accepting only a UUID or `:`, and an id it cannot parse opens an empty editor. `ResultExplorer` keeps a registry of the sessions the manager wrapped, and each open dataset holds one kernel view: opened with `result_open` on the first request or when a link is clicked — concurrent requests share the one open — and closed with `result_close` from `closeDataset`. A request for a dataset after its `closeDataset` is refused rather than opening a view nobody would close; only a link (`reopen`) opens it again. +- **A call is never made to a session that is down.** `callMethod` on a session with no UI comm waits forever, so a Data Explorer asking a kernel that has exited would spin. The session wrapper reports `onDidEndSession` and the runtime state to `setRunning`: while a session is down every call fails at once, and on going down or coming back its open explorers are sent `schema_update`, so they ask for their state again — they show that the session is not running, then, once it is ready, the result or the kernel's reason it is gone. A `get_state` for a result that cannot be opened answers a disconnected state carrying the message, which the Data Explorer shows in place of the table. +- A kernel that predates the results API answers `call_method` with `null`; `call` turns that into a message saying the kernel is too old, rather than a property read on `null`. - `handler.ts` dispatches the requests. Row filters and sort keys are remembered as last set, for `get_state`; filters are ANDed by the kernel, so `supports_conditions` is unsupported, and a filter that cannot be translated or that the kernel rejects goes back with `is_valid: false` and its `error_message`. `get_data_values` makes one `result_page` per distinct row spec. `get_column_profiles` replies `{}` at once and answers through the registration's `sendUiEvent` as `return_column_profiles`. `convert_to_code` is the kernel's `result_sql`, written in the user's own dialect. Column filters and row labels are unsupported. - `filters.ts` maps Positron's `RowFilter`s and sort keys to the kernel's and runs `search_schema` over the columns `result_open` returned. `profiles.ts` maps profile requests and results; the kernel computes one histogram and one frequency table per call, so a large variant gets a call of its own. `format.ts` turns a kernel cell into Positron's `ColumnValue`: a null and, in a floating column only, `NaN`/`inf`/`-inf` become Positron's special-value codes. - `protocol.ts` is a hand-written subset of Positron's protocol types, not a copy of Positron's generated (Elastic-licensed) file, plus the kernel's API types. @@ -138,7 +142,7 @@ A console query result opens in Positron's Data Explorer, served by a backend th **The link** is added by the extension, not the kernel. In a Positron console the kernel marks each tabular `execute_result` with `{"ggsql": {"result": ""}}` in its metadata, which the supervisor exposes as the Result message's `outputMetadata`. `createSession` and `restoreSession` return the supervisor's session through `withResultLinks` (`src/resultLinks.ts`), a `Proxy` that: - **re-fires `onDidReceiveRuntimeMessage`**, in order, and after a marked Result in a *console-mode* session fires a synthesized stdout stream message (same `parent_id` and `when`, an id derived from the result's) holding an OSC 8 hyperlink to `x-ggsql-result:/`. Notebook and background sessions get no link: a notebook has no console to click in. -- **adds `openResource`**, which the supervisor's session does not implement. A clicked link opens the kernel view first — for the title, and so an expired result is reported as a warning before a tab opens — then calls `positron.dataExplorer.open`. Positron offers a clicked console hyperlink only to the *last active* console of each language, so the link carries the session id and is routed by it rather than to the session that was asked. Only a hex key is accepted. +- **adds `openResource`**, which the supervisor's session does not implement. A clicked link opens the kernel view first — for the title, and so an expired result is reported as a warning before a tab opens — then calls `positron.dataExplorer.open`; if that fails, the view is closed again, since no Data Explorer would. Positron offers a clicked console hyperlink only to the *last active* console of each language, so the link carries the session id and is routed by it rather than to the session that was asked. Only a hex key is accepted. - binds everything else to the underlying session, and on `dispose` releases its own subscriptions and the registry entry. When `positron.dataExplorer` is missing (a Positron without extension-hosted Data Explorer backends) nothing is registered and sessions are returned unwrapped, so no link is shown that could not be served. diff --git a/ggsql-vscode/package.json b/ggsql-vscode/package.json index 11114e4fe..d7e50becd 100644 --- a/ggsql-vscode/package.json +++ b/ggsql-vscode/package.json @@ -169,6 +169,7 @@ "type": "integer", "default": 10, "minimum": 1, + "maximum": 4294967295, "markdownDescription": "How many rows of a query result the ggsql console shows. A result with up to twice this many rows is shown in full; a longer one shows this many, then a count of the rest. Every result has a link beneath it that opens the whole result in the Data Explorer. Notebooks are unaffected. Takes effect in consoles started after the change." }, "ggsql.enableSqlFiles": { diff --git a/ggsql-vscode/src/dataExplorer/handler.ts b/ggsql-vscode/src/dataExplorer/handler.ts index 5561b6c1e..468bd0249 100644 --- a/ggsql-vscode/src/dataExplorer/handler.ts +++ b/ggsql-vscode/src/dataExplorer/handler.ts @@ -7,6 +7,11 @@ * session that ran the query. A dataset id is `/`, and * each open dataset holds one kernel view, opened on first use and closed * with the Data Explorer. + * + * A call is only made to a session that is running: one made while the + * kernel is down would wait for an answer that never comes, leaving the Data + * Explorer spinning. The session wrapper (`resultLinks.ts`) reports when a + * session ends and when it is ready again. */ import type * as positron from '@posit-dev/positron'; @@ -148,9 +153,23 @@ function disconnectedState(id: string, message: string): BackendState { }; } +/** Why a session's results cannot be reached while it is down. */ +const SESSION_DOWN = 'The ggsql session that ran this query is not running.'; + +/** What a kernel that predates the results API answers its calls with. */ +const KERNEL_TOO_OLD = + 'The ggsql kernel running this session is too old to open query results in the Data Explorer. ' + + 'Update ggsql and start a new console.'; + export class ResultExplorer { private readonly sessions = new Map(); + /** Sessions whose kernel is down: exited, or restarting. */ + private readonly down = new Set(); private readonly datasets = new Map>(); + /** Datasets with an open Data Explorer, whether or not their view opened. */ + private readonly shown = new Set(); + /** Datasets whose Data Explorer has closed; a late request must not reopen them. */ + private readonly closed = new Set(); constructor( private readonly sendUiEvent: (event: positron.DataExplorerUiEvent) => void, @@ -165,11 +184,28 @@ export class ResultExplorer { if (this.sessions.get(sessionId) === session) { this.sessions.delete(sessionId); } + this.down.delete(sessionId); this.forgetViews(sessionId); }, }; } + /** + * Record whether a session's kernel is running. Going down or coming back + * both mean its views are gone, so its open Data Explorers ask again: they + * show that the session is down, then, once it is back, the kernel's own + * account of the result. + */ + setRunning(sessionId: string, running: boolean): void { + const changed = running ? this.down.delete(sessionId) : !this.down.has(sessionId); + if (!running) { + this.down.add(sessionId); + } + if (changed) { + this.forgetViews(sessionId); + } + } + /** * Forget the views of a session whose kernel has exited or restarted, * since they went with it, and tell each open Data Explorer its schema @@ -178,7 +214,7 @@ export class ResultExplorer { * of why it is gone, rather than a sort or filter that silently did nothing. */ forgetViews(sessionId: string): void { - for (const id of [...this.datasets.keys()]) { + for (const id of this.shown) { if (parseDatasetId(id)?.sessionId === sessionId) { this.datasets.delete(id); this.sendUiEvent({ uri: id, method: 'schema_update', params: {} }); @@ -186,8 +222,21 @@ export class ResultExplorer { } } + /** + * The dataset `id` for a Data Explorer being opened now, even one closed + * before. Concurrent opens of one dataset share one kernel view. + */ + reopen(id: string): Promise { + this.closed.delete(id); + return this.open(id); + } + /** The dataset `id`, opening a kernel view of it the first time. */ open(id: string): Promise { + if (this.closed.has(id)) { + return Promise.reject(new Error('This Data Explorer has been closed.')); + } + this.shown.add(id); let dataset = this.datasets.get(id); if (!dataset) { dataset = this.openView(id); @@ -221,6 +270,8 @@ export class ResultExplorer { closeDataset(id: string): void { const dataset = this.datasets.get(id); this.datasets.delete(id); + this.shown.delete(id); + this.closed.add(id); dataset ?.then(ds => this.call(ds.sessionId, 'result_close', { view: ds.result.view })) .catch(err => this.log(`Closing ${id} failed: ${errorMessage(err)}`)); @@ -228,10 +279,16 @@ export class ResultExplorer { private async call(sessionId: string, method: string, params: object): Promise { const session = this.sessions.get(sessionId); - if (!session?.callMethod) { - throw new Error('The ggsql session that ran this query is no longer running.'); + if (!session?.callMethod || this.down.has(sessionId)) { + throw new Error(SESSION_DOWN); + } + const answer = await session.callMethod(method, params); + // Every results call answers with an object; a kernel without the + // results API answers any call with null. + if (answer === null || answer === undefined) { + throw new Error(KERNEL_TOO_OLD); } - return await session.callMethod(method, params) as T; + return answer as T; } async handleRpc(request: positron.DataExplorerRpcRequest): Promise { diff --git a/ggsql-vscode/src/manager.ts b/ggsql-vscode/src/manager.ts index d3c430fdc..e6bce461e 100644 --- a/ggsql-vscode/src/manager.ts +++ b/ggsql-vscode/src/manager.ts @@ -544,11 +544,24 @@ export function generateMetadata( * flag. A value the kernel would reject is left out, so it uses its default. */ export function consoleRowsEnv(rows: unknown): Record { - return Number.isInteger(rows) && (rows as number) >= 1 + return Number.isInteger(rows) && (rows as number) >= 1 && (rows as number) <= MAX_CONSOLE_ROWS ? { GGSQL_CONSOLE_ROWS: String(rows) } : {}; } +/** The most rows the kernel accepts for `--console-rows`, a `u32`. */ +export const MAX_CONSOLE_ROWS = 4294967295; + +/** + * The kernel environment that turns on keeping query results, which the + * kernel only does when asked. A console session whose results this + * extension can open in the Data Explorer asks; any other session would hold + * results nothing explores. + */ +export function keepResultsEnv(keep: boolean): Record { + return keep ? { GGSQL_KEEP_RESULTS: '1' } : {}; +} + /** * Create a Jupyter kernel spec for ggsql-jupyter * @@ -560,11 +573,13 @@ export function consoleRowsEnv(rows: unknown): Record { * @param kernelPath - Path to the ggsql-jupyter executable * @param readerUri - Data source the kernel should open, if not the default * @param sessionMode - What kind of session this is, when known + * @param keepResults - Whether the kernel should keep results for the Data Explorer */ function createKernelSpec( kernelPath: string, readerUri?: string, - sessionMode?: positron.LanguageRuntimeSessionMode + sessionMode?: positron.LanguageRuntimeSessionMode, + keepResults = false ): JupyterKernelSpec { const argv = [kernelPath, '-f', '{connection_file}']; if (readerUri) { @@ -584,6 +599,7 @@ function createKernelSpec( env: { RUST_LOG: 'error', ...consoleRowsEnv(vscode.workspace.getConfiguration('ggsql').get('consoleRows')), + ...keepResultsEnv(keepResults), }, kernel_protocol_version: '5.3', }; @@ -872,7 +888,9 @@ export class GgsqlRuntimeManager implements positron.LanguageRuntimeManager { const kernelSpec = createKernelSpec( runtimeMetadata.runtimePath, undefined, - sessionMetadata.sessionMode + sessionMetadata.sessionMode, + this._resultExplorer !== undefined && + sessionMetadata.sessionMode === ('console' as positron.LanguageRuntimeSessionMode) ); const dynState = createDynState(); diff --git a/ggsql-vscode/src/resultLinks.ts b/ggsql-vscode/src/resultLinks.ts index ffdf32a57..c54119af4 100644 --- a/ggsql-vscode/src/resultLinks.ts +++ b/ggsql-vscode/src/resultLinks.ts @@ -60,23 +60,41 @@ export function linkMessage( }; } -/** Open a result in the Data Explorer. */ -async function openResult(explorer: ResultExplorer, sessionId: string, key: string): Promise { +/** Show a dataset in a Data Explorer: Positron's `dataExplorer.open`. */ +export type OpenDataExplorer = (options: { providerId: string; datasetId: string; displayName: string }) => Thenable; + +function positronOpen(): OpenDataExplorer | undefined { const api = getPositronApi(); - if (!api) { + return api ? options => api.dataExplorer.open(options) : undefined; +} + +/** Open a result in the Data Explorer. */ +async function openResult( + explorer: ResultExplorer, + sessionId: string, + key: string, + show: OpenDataExplorer | undefined +): Promise { + if (!show) { return; } const id = datasetId(sessionId, key); + let opened = false; try { // Opened here rather than on the Data Explorer's first request, for // the title, and so an expired result is reported before a tab opens. - const dataset = await explorer.open(id); - await api.dataExplorer.open({ + const dataset = await explorer.reopen(id); + opened = true; + await show({ providerId: PROVIDER_ID, datasetId: id, displayName: dataset.result.title, }); } catch (err) { + if (opened) { + // No Data Explorer will close the view, so release it here. + explorer.closeDataset(id); + } const message = errorMessage(err); log(`Opening result ${key} in session ${sessionId} failed: ${message}`); vscode.window.showWarningMessage( @@ -96,7 +114,8 @@ async function openResult(explorer: ResultExplorer, sessionId: string, key: stri */ export function withResultLinks( session: positron.LanguageRuntimeSession, - explorer: ResultExplorer + explorer: ResultExplorer, + show: OpenDataExplorer | undefined = positronOpen() ): positron.LanguageRuntimeSession { const sessionId = session.metadata.sessionId; const isConsole = session.metadata.sessionMode === ('console' as positron.LanguageRuntimeSessionMode); @@ -112,8 +131,16 @@ export function withResultLinks( messages.fire(linkMessage(message, sessionId, key)); } }), - // The kernel's views end with it. - session.onDidEndSession(() => explorer.forgetViews(sessionId)), + // The kernel's views end with it, and until it is back a call would + // never be answered. + session.onDidEndSession(() => explorer.setRunning(sessionId, false)), + session.onDidChangeRuntimeState(state => { + if (state === ('ready' as positron.RuntimeState) || state === ('idle' as positron.RuntimeState)) { + explorer.setRunning(sessionId, true); + } else if (state === ('exited' as positron.RuntimeState)) { + explorer.setRunning(sessionId, false); + } + }), ]; const openResource = async (resource: vscode.Uri | string): Promise => { @@ -124,7 +151,7 @@ export function withResultLinks( // Positron offers the link only to the most recently active ggsql // console, which need not be the one that printed it; the session id // routes it back to the right one. - void openResult(explorer, link.sessionId, link.key); + void openResult(explorer, link.sessionId, link.key, show); return true; }; diff --git a/ggsql-vscode/src/test/dataExplorer.test.ts b/ggsql-vscode/src/test/dataExplorer.test.ts index 128fd7798..f8554cec7 100644 --- a/ggsql-vscode/src/test/dataExplorer.test.ts +++ b/ggsql-vscode/src/test/dataExplorer.test.ts @@ -122,7 +122,28 @@ suite('Data Explorer backend', () => { test('a session that is gone is an error, not a hang', async () => { const explorer = new ResultExplorer(() => { }); const r = await explorer.handleRpc({ method: 'get_schema', uri: ID, params: { column_indices: [0] } }); - assert.match(r.error_message ?? '', /no longer running/); + assert.match(r.error_message ?? '', /not running/); + }); + + test('a session that is down is not called, so nothing waits on it', async () => { + const { explorer, calls, rpc } = setup(); + explorer.setRunning('ggsql-abc', false); + const r = await rpc('get_schema', { column_indices: [0] }); + assert.match(r.error_message ?? '', /not running/); + assert.deepStrictEqual(calls, []); + }); + + test('a kernel without the results API is reported as too old', async () => { + const explorer = new ResultExplorer(() => { }); + explorer.addSession('ggsql-abc', { callMethod: async () => null }); + const r = await explorer.handleRpc({ method: 'get_state', uri: ID, params: {} }); + assert.match(String((r.result as Record).error_message), /too old/); + }); + + test('concurrent requests share one kernel view', async () => { + const { calls, rpc } = setup(); + await Promise.all([rpc('get_state'), rpc('get_schema', { column_indices: [0] }), rpc('get_state')]); + assert.deepStrictEqual(calls, [['result_open', { key: '0f9e' }]]); }); test('get_data_values batches columns that ask for the same rows', async () => { @@ -292,13 +313,17 @@ suite('Data Explorer backend', () => { assert.match((await rpc('nope')).error_message ?? '', /Unknown method/); }); - test('closing a dataset closes its view, and the next request reopens', async () => { + test('closing a dataset closes its view, and a late request does not reopen it', async () => { const { explorer, calls, rpc } = setup({ result_close: () => ({}) }); await rpc('get_state'); explorer.closeDataset(ID); await new Promise(resolve => setTimeout(resolve, 0)); assert.deepStrictEqual(calls.at(-1), ['result_close', { view: 'v1' }]); - await rpc('get_state'); + const late = await rpc('get_schema', { column_indices: [0] }); + assert.match(late.error_message ?? '', /closed/); + assert.strictEqual(calls.filter(c => c[0] === 'result_open').length, 1); + // Opening it again is what reopens it. + await explorer.reopen(ID); assert.strictEqual(calls.filter(c => c[0] === 'result_open').length, 2); }); }); diff --git a/ggsql-vscode/src/test/manager.test.ts b/ggsql-vscode/src/test/manager.test.ts index 4caff9db0..8805ab92b 100644 --- a/ggsql-vscode/src/test/manager.test.ts +++ b/ggsql-vscode/src/test/manager.test.ts @@ -1,6 +1,6 @@ import * as assert from 'assert'; import * as vscode from 'vscode'; -import { GgsqlRuntimeManager, consoleRowsEnv, createDynState, getSupervisorApi } from '../manager'; +import { GgsqlRuntimeManager, MAX_CONSOLE_ROWS, consoleRowsEnv, createDynState, getSupervisorApi, keepResultsEnv } from '../manager'; suite('manager', () => { // The suites run in stock VS Code, where positron.positron-supervisor is @@ -19,9 +19,20 @@ suite('manager', () => { test('consoleRowsEnv leaves out a value the kernel would refuse', () => { // The kernel exits on a bad value, so the session would fail to start. - for (const rows of [0, -1, 2.5, '10', undefined, null]) { + for (const rows of [0, -1, 2.5, '10', undefined, null, MAX_CONSOLE_ROWS + 1, 1e21]) { assert.deepStrictEqual(consoleRowsEnv(rows), {}, String(rows)); } + assert.deepStrictEqual(consoleRowsEnv(MAX_CONSOLE_ROWS), { GGSQL_CONSOLE_ROWS: String(MAX_CONSOLE_ROWS) }); + }); + + test('the setting\'s maximum is the kernel\'s', () => { + const pkg = require('../../package.json'); + assert.strictEqual(pkg.contributes.configuration.properties['ggsql.consoleRows'].maximum, MAX_CONSOLE_ROWS); + }); + + test('keepResultsEnv turns keeping results on only when asked', () => { + assert.deepStrictEqual(keepResultsEnv(true), { GGSQL_KEEP_RESULTS: '1' }); + assert.deepStrictEqual(keepResultsEnv(false), {}); }); test('createDynState falls back to the default session name', () => { diff --git a/ggsql-vscode/src/test/resultLinks.test.ts b/ggsql-vscode/src/test/resultLinks.test.ts index 8e1fdd77a..05d271193 100644 --- a/ggsql-vscode/src/test/resultLinks.test.ts +++ b/ggsql-vscode/src/test/resultLinks.test.ts @@ -50,12 +50,20 @@ function result(id: string, key?: unknown): Message { function fakeSession(sessionMode: string) { const messages = new vscode.EventEmitter(); const ended = new vscode.EventEmitter(); + const state = new vscode.EventEmitter(); + const calls: string[] = []; let disposed = false; const session = { metadata: { sessionId: 'ggsql-abc', sessionMode }, onDidReceiveRuntimeMessage: messages.event, onDidEndSession: ended.event, - callMethod: async () => ({}), + onDidChangeRuntimeState: state.event, + callMethod: async (method: string) => { + calls.push(method); + return method === 'result_open' + ? { view: 'v1', key: '0f9e', title: 'Query [1]', query: '', columns: [], num_rows: 1 } + : {}; + }, dispose() { disposed = true; }, @@ -67,10 +75,15 @@ function fakeSession(sessionMode: string) { session: session as unknown as positron.LanguageRuntimeSession, raw: session, fire: (m: positron.LanguageRuntimeMessage) => messages.fire(m), + end: () => ended.fire({ reason: 'restart' } as positron.LanguageRuntimeExit), + setState: (s: string) => state.fire(s as positron.RuntimeState), + calls, disposed: () => disposed, }; } +const tick = () => new Promise(resolve => setTimeout(resolve, 0)); + function received(sessionMode: string, input: Message[]): Message[] { const fake = fakeSession(sessionMode); const wrapped = withResultLinks(fake.session, new ResultExplorer(() => { })); @@ -125,6 +138,68 @@ suite('withResultLinks', () => { assert.ok(fake.disposed()); }); + test('clicking a link opens a view and shows it under the result\'s title', async () => { + const fake = fakeSession('console'); + const shown: object[] = []; + const wrapped = withResultLinks(fake.session, new ResultExplorer(() => { }), async options => { + shown.push(options); + }); + assert.strictEqual(await wrapped.openResource!('x-ggsql-result:ggsql-abc/0f9e'), true); + await tick(); + assert.deepStrictEqual(fake.calls, ['result_open']); + assert.deepStrictEqual(shown, [{ providerId: 'ggsql', datasetId: 'ggsql:ggsql-abc/0f9e', displayName: 'Query [1]' }]); + wrapped.dispose(); + }); + + test('a view whose Data Explorer fails to open is closed again', async () => { + const fake = fakeSession('console'); + const wrapped = withResultLinks(fake.session, new ResultExplorer(() => { }), async () => { + throw new Error('no editor'); + }); + await wrapped.openResource!('x-ggsql-result:ggsql-abc/0f9e'); + await tick(); + await tick(); + assert.deepStrictEqual(fake.calls, ['result_open', 'result_close']); + wrapped.dispose(); + }); + + test('a link reopens a result whose Data Explorer was closed', async () => { + const fake = fakeSession('console'); + const explorer = new ResultExplorer(() => { }); + const wrapped = withResultLinks(fake.session, explorer, async () => { }); + await wrapped.openResource!('x-ggsql-result:ggsql-abc/0f9e'); + await tick(); + explorer.closeDataset('ggsql:ggsql-abc/0f9e'); + await tick(); + await wrapped.openResource!('x-ggsql-result:ggsql-abc/0f9e'); + await tick(); + assert.deepStrictEqual(fake.calls, ['result_open', 'result_close', 'result_open']); + wrapped.dispose(); + }); + + test('a session that ended answers at once, and again once it is ready', async () => { + const fake = fakeSession('console'); + const events: string[] = []; + const explorer = new ResultExplorer(e => events.push(e.method)); + const wrapped = withResultLinks(fake.session, explorer, async () => { }); + const getState = () => explorer.handleRpc({ method: 'get_state', uri: 'ggsql:ggsql-abc/0f9e', params: {} }); + await getState(); + + fake.end(); + assert.deepStrictEqual(events, ['schema_update']); + const down = (await getState()).result as Record; + assert.strictEqual(down.connected, false); + assert.match(String(down.error_message), /not running/); + assert.deepStrictEqual(fake.calls, ['result_open']); + + fake.setState('ready'); + assert.deepStrictEqual(events, ['schema_update', 'schema_update']); + const up = (await getState()).result as Record; + assert.notStrictEqual(up.connected, false); + assert.deepStrictEqual(fake.calls, ['result_open', 'result_open']); + wrapped.dispose(); + }); + test('resultKey reads only result messages', () => { assert.strictEqual(resultKey(result('r', 'ab12')), 'ab12'); assert.strictEqual(resultKey(message('o', 'output', { outputMetadata: { ggsql: { result: 'ab' } } })), undefined); diff --git a/src/AGENTS.md b/src/AGENTS.md index d99f9d860..ea367145a 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -59,7 +59,7 @@ Grammar lives in [`/tree-sitter-ggsql/`](../tree-sitter-ggsql/) — when adding `SqlDialect` trait in `mod.rs` lets each driver supply its own type names, information-schema queries, and spatial helper methods (`sql_st_transform`, `sql_geometry_to_wkb`, `sql_geometry_bbox`, `sql_ensure_geometry`, `sql_select_replace`, `sql_spatial_setup`). -**Caching layer.** `CachingReader` (`cache.rs`) wraps a primary reader plus an in-memory `CacheBackend`, splitting work across two `Reader` surfaces. **`execute_sql` = source**: base reads of the user's data plus user setup/DML run on the primary (with result memoization), except `ggsql:` builtins, the `__ggsql_cache_meta__` table, and reads that reference a cache-resident internal table, which go to the cache. **`execute_sql_cached` = compute**: all dialect-generated/derived SQL (schema probes, stats, projection/map transforms, spatial setup, final layer queries — everything operating on `__ggsql_*` tables) runs on the cache; it defaults to `execute_sql` so a plain reader runs everything on one connection. Cache routing is by **exact-identifier membership** in the set of tables registered into the cache. Memoization keys on `hash(primary_uri + sql)` and is tracked in the `__ggsql_cache_meta__` table inside the cache backend. Each memoized read is bounded by a **TTL** (default 300s) and the whole memo by an **LRU byte budget** (default 512 MB); both are configurable via `CacheConfig` (env `GGSQL_CACHE_DISABLED`/`GGSQL_CACHE_TTL`/`GGSQL_CACHE_MAX_BYTES`, or per-connection URI query parameters `?cache_ttl=…&cache_max_bytes=…&cache_disabled=…`). The `__ggsql_cache_meta__` table is queryable for introspection (`SELECT * FROM __ggsql_cache_meta__`). Pure/non-visual SQL (CLI table fallback, Jupyter) goes through `execute_sql` so it reads the primary rather than the empty cache. `Reader::materialize_table` (default = `CREATE TEMP TABLE` on the reader, no Rust roundtrip) is overridden to read the body via the source surface and `register()` the result into the cache, so the primary is never written to; `Reader::caches_sources()` (default `false`, `true` for `CachingReader`) gates the executor's per-layer source staging: file sources are staged on the cache surface, while identifiers go through `materialize_table`, which routes the read to the cache (CTEs, builtins, cache-resident tables) or the primary as needed. `dialect()` returns the **cache** dialect, and every compute-surface failure is prefixed with the cache backend's scheme (``on the `duckdb` cache backend: …``) so a cache-dialect driver error is not mistaken for one from the user's own connection. Selected via the composite `+://` scheme (`reader_from_uri` / `split_cache_uri`) or the CLI `--cache` flag; off by default. The memo's bookkeeping — registering a table, its row in `__ggsql_cache_meta__`, LRU eviction under the byte budget — is `CacheTables`, which other callers use on a backend of their own (the Jupyter kernel's results API does, without wrapping the user's reader). An entry can be **pinned**, exempting it from eviction until unpinned; `CachingReader` never pins, and the TTL check stays in `CachingReader`. +**Caching layer.** `CachingReader` (`cache.rs`) wraps a primary reader plus an in-memory `CacheBackend`, splitting work across two `Reader` surfaces. **`execute_sql` = source**: base reads of the user's data plus user setup/DML run on the primary (with result memoization), except `ggsql:` builtins, the `__ggsql_cache_meta__` table, and reads that reference a cache-resident internal table, which go to the cache. **`execute_sql_cached` = compute**: all dialect-generated/derived SQL (schema probes, stats, projection/map transforms, spatial setup, final layer queries — everything operating on `__ggsql_*` tables) runs on the cache; it defaults to `execute_sql` so a plain reader runs everything on one connection. Cache routing is by **exact-identifier membership** in the set of tables registered into the cache. Memoization keys on `hash(primary_uri + sql)` and is tracked in the `__ggsql_cache_meta__` table inside the cache backend. Each memoized read is bounded by a **TTL** (default 300s) and the whole memo by an **LRU byte budget** (default 512 MB); both are configurable via `CacheConfig` (env `GGSQL_CACHE_DISABLED`/`GGSQL_CACHE_TTL`/`GGSQL_CACHE_MAX_BYTES`, or per-connection URI query parameters `?cache_ttl=…&cache_max_bytes=…&cache_disabled=…`). The `__ggsql_cache_meta__` table is queryable for introspection (`SELECT * FROM __ggsql_cache_meta__`). Pure/non-visual SQL (CLI table fallback, Jupyter) goes through `execute_sql` so it reads the primary rather than the empty cache. `Reader::materialize_table` (default = `CREATE TEMP TABLE` on the reader, no Rust roundtrip) is overridden to read the body via the source surface and `register()` the result into the cache, so the primary is never written to; `Reader::caches_sources()` (default `false`, `true` for `CachingReader`) gates the executor's per-layer source staging: file sources are staged on the cache surface, while identifiers go through `materialize_table`, which routes the read to the cache (CTEs, builtins, cache-resident tables) or the primary as needed. `dialect()` returns the **cache** dialect, and every compute-surface failure is prefixed with the cache backend's scheme (``on the `duckdb` cache backend: …``) so a cache-dialect driver error is not mistaken for one from the user's own connection. Selected via the composite `+://` scheme (`reader_from_uri` / `split_cache_uri`) or the CLI `--cache` flag; off by default. The memo's bookkeeping — registering a table, its row in `__ggsql_cache_meta__`, LRU eviction under the byte budget — is `CacheTables`, which other callers use on a backend of their own (the Jupyter kernel's results API does, without wrapping the user's reader). An entry can be **pinned**, exempting it from eviction until unpinned (pins are kept in memory, not in the metadata table); `CachingReader` never pins, and the TTL and enabled checks stay in `CachingReader`. `CacheTables` takes a byte budget rather than a `CacheConfig`, and `evict_to`/`total_bytes` let a caller with a wider budget evict to that instead. ### `execute/` diff --git a/src/reader/cache.rs b/src/reader/cache.rs index 1eda4fc38..aef3d03fe 100644 --- a/src/reader/cache.rs +++ b/src/reader/cache.rs @@ -119,7 +119,7 @@ fn now_ms() -> i64 { /// Estimate the in-memory size of a DataFrame as the sum of its Arrow columns' /// memory footprints. -fn estimate_bytes(df: &DataFrame) -> i64 { +pub fn estimate_bytes(df: &DataFrame) -> i64 { df.get_columns() .iter() .map(|col| col.get_array_memory_size()) @@ -138,7 +138,8 @@ pub struct MemoEntry { /// `__ggsql_cache_meta__` table, which records its size and when it was last /// used. Once the entries' total size passes the budget the least recently /// used are dropped, except entries that are **pinned**: those stay until -/// unpinned, even past the budget. +/// unpinned, even past the budget. Pins are kept here rather than in the +/// metadata table, so they never show in it. /// /// [`CachingReader`] keeps memoized reads here, keyed by a hash of their SQL, /// and never pins. Other callers can keep tables in a `CacheTables` of their @@ -147,52 +148,40 @@ pub struct MemoEntry { pub struct CacheTables { /// The in-memory backend holding the tables and the metadata table. backend: Box, - /// TTL + byte-budget configuration. Checking the TTL is the caller's job. - config: CacheConfig, + /// The byte budget `put` and unpinning evict to. + max_bytes: u64, + /// Keys of the pinned entries. + pinned: RefCell>, /// Whether the metadata table has been created in the backend. meta_ready: Cell, } impl CacheTables { - pub fn new(backend: Box, config: CacheConfig) -> Self { + /// Tables in `backend`, evicted to `max_bytes`. + pub fn new(backend: Box, max_bytes: u64) -> Self { Self { backend, - config, + max_bytes, + pinned: RefCell::new(HashSet::new()), meta_ready: Cell::new(false), } } - /// A `CacheTables` on a fresh in-memory backend: DuckDB when the `duckdb` - /// feature is compiled in, SQLite otherwise. - #[cfg(any(feature = "duckdb", feature = "sqlite"))] - pub fn in_memory(config: CacheConfig) -> Result { - use crate::reader::connection::{build_reader, cache_uri, default_cache_scheme}; - let backend = build_reader(cache_uri(default_cache_scheme())?)?; - Ok(Self::new(backend, config)) - } - /// The backend holding the tables, for queries over them. pub fn backend(&self) -> &dyn Reader { &*self.backend } - /// The active configuration. - pub fn config(&self) -> &CacheConfig { - &self.config - } - /// Create the metadata table in the backend if it doesn't exist yet. - pub fn ensure_meta_table(&self) -> Result<()> { + fn ensure_meta_table(&self) -> Result<()> { if self.meta_ready.get() { return Ok(()); } - // `pinned` is an integer rather than a BOOLEAN so the same DDL and - // comparisons work on every cache backend. let sql = format!( "CREATE TABLE IF NOT EXISTS {} (\ cache_key VARCHAR PRIMARY KEY, sql VARCHAR NOT NULL, table_name VARCHAR NOT NULL, \ fetched_at_epoch_ms BIGINT NOT NULL, last_accessed_epoch_ms BIGINT NOT NULL, \ - byte_estimate BIGINT NOT NULL, row_count BIGINT NOT NULL, pinned BIGINT NOT NULL)", + byte_estimate BIGINT NOT NULL, row_count BIGINT NOT NULL)", naming::quote_ident(naming::CACHE_META_TABLE) ); self.backend.execute_sql(&sql)?; @@ -235,13 +224,18 @@ impl CacheTables { let byte_estimate = estimate_bytes(&df); let row_count = df.height() as i64; self.backend.register(table, df, true)?; - if let Err(e) = self.insert_meta(key, sql, table, byte_estimate, row_count, pinned) { + if let Err(e) = self.insert_meta(key, sql, table, byte_estimate, row_count) { // The table registered but the meta row didn't: drop the orphan so // it can't leak, then surface the error. let _ = self.backend.unregister(table); return Err(e); } - self.evict_over_budget() + if pinned { + self.pinned.borrow_mut().insert(key.to_string()); + } else { + self.pinned.borrow_mut().remove(key); + } + self.evict_to(self.max_bytes) } /// Record an entry in the metadata table. @@ -252,7 +246,6 @@ impl CacheTables { table: &str, byte_estimate: i64, row_count: i64, - pinned: bool, ) -> Result<()> { let now = now_ms(); // Upsert as DELETE + INSERT rather than INSERT OR REPLACE: the cache @@ -268,8 +261,8 @@ impl CacheTables { let stmt = format!( "INSERT INTO {} \ (cache_key, sql, table_name, fetched_at_epoch_ms, last_accessed_epoch_ms, \ - byte_estimate, row_count, pinned) \ - VALUES ({}, {}, {}, {}, {}, {}, {}, {})", + byte_estimate, row_count) \ + VALUES ({}, {}, {}, {}, {}, {}, {})", naming::quote_ident(naming::CACHE_META_TABLE), naming::quote_literal(key), naming::quote_literal(sql), @@ -278,14 +271,14 @@ impl CacheTables { now, byte_estimate, row_count, - i64::from(pinned), ); self.backend.execute_sql(&stmt)?; Ok(()) } /// Advance the last-accessed timestamp for `key` (LRU bookkeeping). - pub fn touch(&self, key: &str) -> Result<()> { + pub(crate) fn touch(&self, key: &str) -> Result<()> { + self.ensure_meta_table()?; let stmt = format!( "UPDATE {} SET last_accessed_epoch_ms = {} WHERE cache_key = {}", naming::quote_ident(naming::CACHE_META_TABLE), @@ -300,23 +293,19 @@ impl CacheTables { /// Pinned entries are never evicted. Unpinning evicts over the budget, /// since the entry counts towards it again. pub fn set_pinned(&self, key: &str, pinned: bool) -> Result<()> { - self.ensure_meta_table()?; - let stmt = format!( - "UPDATE {} SET pinned = {}, last_accessed_epoch_ms = {} WHERE cache_key = {}", - naming::quote_ident(naming::CACHE_META_TABLE), - i64::from(pinned), - now_ms(), - naming::quote_literal(key), - ); - self.backend.execute_sql(&stmt)?; - if !pinned { - self.evict_over_budget()?; + self.touch(key)?; + if pinned { + self.pinned.borrow_mut().insert(key.to_string()); + Ok(()) + } else { + self.pinned.borrow_mut().remove(key); + self.evict_to(self.max_bytes) } - Ok(()) } /// Drop a single entry: unregister the table, then delete its meta row. - pub fn drop_entry(&self, key: &str, table: &str) -> Result<()> { + pub(crate) fn drop_entry(&self, key: &str, table: &str) -> Result<()> { + self.ensure_meta_table()?; self.backend.unregister(table)?; let del = format!( "DELETE FROM {} WHERE cache_key = {}", @@ -324,33 +313,44 @@ impl CacheTables { naming::quote_literal(key), ); self.backend.execute_sql(&del)?; + self.pinned.borrow_mut().remove(key); Ok(()) } - /// Evict unpinned LRU entries until the cumulative byte estimate is within - /// `max_bytes`, or nothing unpinned is left. - pub fn evict_over_budget(&self) -> Result<()> { + /// The entries' total byte estimate, pinned ones included. + pub fn total_bytes(&self) -> Result { + self.ensure_meta_table()?; // Cast the SUM to BIGINT. - let sum_sql = format!( + let df = self.backend.execute_sql(&format!( "SELECT CAST(COALESCE(SUM(byte_estimate), 0) AS BIGINT) AS n FROM {}", naming::quote_ident(naming::CACHE_META_TABLE) - ); - loop { - let df = self.backend.execute_sql(&sum_sql)?; - let total = if df.height() == 0 { - 0 + ))?; + let total = if df.height() == 0 { + 0 + } else { + as_i64(df.column("n")?)?.value(0) + }; + Ok(total.max(0) as u64) + } + + /// Evict unpinned entries, least recently used first, until the total + /// byte estimate is within `max_bytes` or nothing unpinned is left. A + /// caller with a budget wider than this cache's passes what it allows. + pub fn evict_to(&self, max_bytes: u64) -> Result<()> { + while self.total_bytes()? > max_bytes { + let pinned = self.pinned.borrow(); + let exclude = if pinned.is_empty() { + String::new() } else { - as_i64(df.column("n")?)?.value(0) + let keys: Vec = pinned.iter().map(|k| naming::quote_literal(k)).collect(); + format!(" WHERE cache_key NOT IN ({})", keys.join(", ")) }; - if total <= self.config.max_bytes as i64 { - return Ok(()); - } - let pick = format!( - "SELECT cache_key, table_name FROM {} WHERE pinned = 0 \ + drop(pinned); + let df = self.backend.execute_sql(&format!( + "SELECT cache_key, table_name FROM {}{exclude} \ ORDER BY last_accessed_epoch_ms ASC LIMIT 1", naming::quote_ident(naming::CACHE_META_TABLE) - ); - let df = self.backend.execute_sql(&pick)?; + ))?; if df.height() == 0 { return Ok(()); } @@ -358,10 +358,11 @@ impl CacheTables { let table = as_str(df.column("table_name")?)?.value(0).to_string(); self.drop_entry(&key, &table)?; } + Ok(()) } /// Drop every entry, one at a time. - pub fn clear(&self) -> Result<()> { + pub(crate) fn clear(&self) -> Result<()> { self.ensure_meta_table()?; let df = self.backend.execute_sql(&format!( "SELECT cache_key, table_name FROM {}", @@ -398,6 +399,8 @@ pub struct CachingReader { /// In-memory writeable cache: derived tables, registered data, memoized /// reads, and the memo's bookkeeping. tables: CacheTables, + /// TTL and byte-budget configuration of the memo. + config: CacheConfig, /// Connection URI of the primary. primary_uri: String, /// Scheme of the cache backend as spelled in the connection URI. @@ -442,7 +445,8 @@ impl CachingReader { ) -> Self { Self { primary, - tables: CacheTables::new(cache, config), + tables: CacheTables::new(cache, config.max_bytes), + config, primary_uri: primary_uri.into(), cache_scheme: cache_scheme.into(), resident: RefCell::new(HashSet::new()), @@ -451,7 +455,7 @@ impl CachingReader { /// The active cache configuration. pub fn cache_config(&self) -> &CacheConfig { - self.tables.config() + &self.config } /// The cache backend. @@ -514,7 +518,7 @@ impl Reader for CachingReader { } // With caching disabled the memo is never consulted or written. - if !self.tables.config().enabled { + if !self.config.enabled { return self.primary.execute_sql(sql); } @@ -524,7 +528,7 @@ impl Reader for CachingReader { // it and fall through to a primary re-fetch. if let Some(entry) = self.tables.lookup(&key)? { let age_ms = (now_ms() - entry.fetched_at_epoch_ms).max(0); - let ttl_ms = (self.tables.config().ttl_secs as i64).saturating_mul(1000); + let ttl_ms = (self.config.ttl_secs as i64).saturating_mul(1000); if age_ms < ttl_ms { let select = format!("SELECT * FROM {}", naming::quote_ident(&entry.table_name)); if let Ok(df) = self.cache().execute_sql(&select) { @@ -1418,14 +1422,7 @@ mod behavior_tests { } fn tiny_budget_tables() -> CacheTables { - CacheTables::new( - Box::new(DuckDBReader::new_in_memory().unwrap()), - CacheConfig { - enabled: true, - ttl_secs: 300, - max_bytes: 1, - }, - ) + CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), 1) } #[test] @@ -1460,6 +1457,55 @@ mod behavior_tests { .is_err()); } + #[test] + fn test_a_fresh_cache_answers_every_call() { + // Each call sets up the metadata table itself. + let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), 1); + assert_eq!(tables.total_bytes().unwrap(), 0); + tables.touch("nothing").unwrap(); + tables.set_pinned("nothing", true).unwrap(); + tables.evict_to(0).unwrap(); + assert!(tables.lookup("nothing").unwrap().is_none()); + } + + #[test] + fn test_pins_stay_out_of_the_metadata_table() { + let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), u64::MAX); + tables + .put( + "a", + "a", + "__t_a__", + df! { "x" => vec![1_i64] }.unwrap(), + true, + ) + .unwrap(); + let meta = tables + .backend() + .execute_sql(&format!( + "SELECT * FROM {}", + naming::quote_ident(naming::CACHE_META_TABLE) + )) + .unwrap(); + assert!(!meta.get_column_names().iter().any(|c| c == "pinned")); + } + + #[test] + fn test_evict_to_a_caller_budget_counts_pinned_entries_but_keeps_them() { + let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), u64::MAX); + let frame = || df! { "x" => vec![1_i64, 2, 3] }.unwrap(); + tables.put("a", "a", "__t_a__", frame(), true).unwrap(); + tables.put("b", "b", "__t_b__", frame(), false).unwrap(); + let both = tables.total_bytes().unwrap(); + assert!(both > 0); + + tables.evict_to(0).unwrap(); + assert!(tables.lookup("a").unwrap().is_some()); + assert!(tables.lookup("b").unwrap().is_none()); + // What is left is the pinned entry, still counted. + assert!(tables.total_bytes().unwrap() > 0 && tables.total_bytes().unwrap() < both); + } + #[test] fn test_parse_human_bytes() { assert_eq!(parse_human_bytes("1024"), Some(1024)); diff --git a/src/reader/connection.rs b/src/reader/connection.rs index 79bbca4ce..17e69be66 100644 --- a/src/reader/connection.rs +++ b/src/reader/connection.rs @@ -197,7 +197,7 @@ impl ConnUri { } /// Map a cache-backend scheme to its in-memory connection URI. -pub(crate) fn cache_uri(scheme: &str) -> Result<&'static str> { +fn cache_uri(scheme: &str) -> Result<&'static str> { match scheme { "duckdb" => Ok("duckdb://memory"), "sqlite" => Ok("sqlite://:memory:"), @@ -209,17 +209,6 @@ pub(crate) fn cache_uri(scheme: &str) -> Result<&'static str> { } } -/// The in-memory backend a cache uses when none is named: DuckDB when the -/// `duckdb` feature is compiled in, SQLite otherwise. -#[cfg(any(feature = "duckdb", feature = "sqlite"))] -pub(crate) fn default_cache_scheme() -> &'static str { - if cfg!(feature = "duckdb") { - "duckdb" - } else { - "sqlite" - } -} - /// Build a reader from a non-composite connection URI pub fn build_reader(uri: &str) -> Result> { let conn = ConnUri::parse(uri)?; @@ -603,7 +592,11 @@ fn auto_cache_if_needed( { use crate::reader::cache::CacheConfig; - let cache_scheme = default_cache_scheme(); + let cache_scheme = if cfg!(feature = "duckdb") { + "duckdb" + } else { + "sqlite" + }; let cache = build_reader(cache_uri(cache_scheme)?)?; Ok(Box::new(crate::reader::CachingReader::with_config( reader, diff --git a/src/reader/dialects/duckdb.rs b/src/reader/dialects/duckdb.rs index 9c00be237..7a413a1eb 100644 --- a/src/reader/dialects/duckdb.rs +++ b/src/reader/dialects/duckdb.rs @@ -21,6 +21,11 @@ impl SqlDialect for DuckDbDialect { format!("isfinite({expr})") } + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let flags = if case_sensitive { "" } else { ", 'i'" }; + Some(format!("regexp_matches({expr}, {pattern}{flags})")) + } + fn sql_st_transform(&self, column: &str, source_crs: &str, target_crs: &str) -> String { format!( "ST_Transform({}, '{}', '{}', always_xy := true)", diff --git a/src/reader/dialects/mod.rs b/src/reader/dialects/mod.rs index cbbb375f7..b5f4c7a67 100644 --- a/src/reader/dialects/mod.rs +++ b/src/reader/dialects/mod.rs @@ -303,6 +303,26 @@ mod tests { ); } + #[test] + fn regex_match_only_where_the_backend_has_one() { + let duckdb = dialect_for_scheme("duckdb").unwrap(); + assert_eq!( + duckdb.sql_regex_match("c", "'^a'", false).as_deref(), + Some("regexp_matches(c, '^a', 'i')") + ); + assert_eq!( + dialect_for_scheme("postgres") + .unwrap() + .sql_regex_match("c", "'^a'", true) + .as_deref(), + Some("c ~ '^a'") + ); + assert!(dialect_for_scheme("mysql") + .unwrap() + .sql_regex_match("c", "'^a'", true) + .is_none()); + } + /// Contract for the quantile hook: callers pass the raw (unquoted) /// column name and the dialect quotes it. A name needing quoting must /// appear quoted — interpolating it raw breaks on any real column whose diff --git a/src/reader/dialects/postgres.rs b/src/reader/dialects/postgres.rs index 54e550401..3f124bfe8 100644 --- a/src/reader/dialects/postgres.rs +++ b/src/reader/dialects/postgres.rs @@ -15,6 +15,11 @@ impl SqlDialect for PostgresDialect { true } + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let op = if case_sensitive { "~" } else { "~*" }; + Some(format!("{expr} {op} {pattern}")) + } + fn sql_generate_series(&self, n: usize) -> String { let __ggsql_seq__ = self.quote_ident("__ggsql_seq__"); format!( diff --git a/src/reader/duckdb.rs b/src/reader/duckdb.rs index 7567fa307..2e431120f 100644 --- a/src/reader/duckdb.rs +++ b/src/reader/duckdb.rs @@ -162,24 +162,6 @@ impl DuckDBReader { }) } - /// Append `df`'s rows to the temporary table `name` with DuckDB's Arrow - /// appender. - fn append_rows(&self, name: &str, df: &DataFrame) -> Result<()> { - let fail = |e: duckdb::Error| { - GgsqlError::ReaderError(format!("Failed to append to table '{}': {}", name, e)) - }; - // `register` creates temporary tables, which DuckDB keeps in its - // `temp` catalog. - let mut appender = self - .conn - .appender_to_catalog_and_db(name, "temp", "main") - .map_err(fail)?; - appender - .append_record_batch(df.inner().clone()) - .map_err(fail)?; - appender.flush().map_err(fail) - } - /// Get a reference to the underlying DuckDB connection /// /// Useful for executing setup queries (CREATE TABLE, INSERT, etc.) @@ -302,30 +284,6 @@ impl Reader for DuckDBReader { GgsqlError::ReaderError(format!("Failed to register table '{}': {}", name, e)) })?; - // The rest goes through DuckDB's Arrow appender, which is several - // times faster than one `INSERT` per chunk on large frames (it - // chunks to the vector size itself). A type the appender cannot - // convert falls back to the `INSERT` loop, starting over so a - // partial append never leaves a half-filled table. - if self - .append_rows( - name, - &df.slice(MAX_ARROW_BATCH_ROWS, total_rows - MAX_ARROW_BATCH_ROWS), - ) - .is_ok() - { - self.registered_tables.note_registered(name); - return Ok(()); - } - let params = dataframe_to_arrow_params(&first_chunk)?; - let recreate_sql = format!( - "CREATE OR REPLACE TEMP TABLE {} AS SELECT * FROM arrow(?, ?)", - naming::quote_ident(name) - ); - self.conn.execute(&recreate_sql, params).map_err(|e| { - GgsqlError::ReaderError(format!("Failed to register table '{}': {}", name, e)) - })?; - let mut offset = MAX_ARROW_BATCH_ROWS; while offset < total_rows { let chunk_size = std::cmp::min(MAX_ARROW_BATCH_ROWS, total_rows - offset); @@ -494,30 +452,6 @@ mod tests { ); } - #[test] - fn test_register_large_frame_keeps_every_row_in_order() { - // Past one Arrow chunk, the rest goes through the appender. - let reader = DuckDBReader::from_connection_string("duckdb://memory").unwrap(); - let n = 5000; - let df = df! { - "i" => (0..n as i64).collect::>(), - "s" => (0..n).map(|i| format!("row {i}")).collect::>(), - "f" => (0..n).map(|i| (i % 7 == 0).then_some(i as f64 / 2.0)).collect::>(), - } - .unwrap(); - reader.register("big", df, false).unwrap(); - - let out = reader - .execute_sql("SELECT i, s, f FROM big ORDER BY rowid") - .unwrap(); - assert_eq!(out.height(), n); - let i = as_i64(out.column("i").unwrap()).unwrap(); - let s = as_str(out.column("s").unwrap()).unwrap(); - assert!((0..n).all(|r| i.value(r) == r as i64)); - assert_eq!(s.value(4999), "row 4999"); - assert_eq!(out.column("f").unwrap().null_count(), n - n.div_ceil(7)); - } - #[test] fn test_register_duplicate_name_errors() { let reader = DuckDBReader::from_connection_string("duckdb://memory").unwrap(); diff --git a/src/reader/mod.rs b/src/reader/mod.rs index aa02e9036..469f722bc 100644 --- a/src/reader/mod.rs +++ b/src/reader/mod.rs @@ -450,6 +450,14 @@ pub trait SqlDialect { } } + /// Whether `expr` matches the regular expression in the SQL string + /// literal `pattern`, or `None` when the backend has no regular + /// expressions. Default `None`. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let _ = (expr, pattern, case_sensitive); + None + } + // ===================================================================== // Statement generators and DDL // ===================================================================== From cd982fe975c2e0150223e65ce4764ca0d453538d Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Thu, 19 Nov 2026 15:44:14 -0600 Subject: [PATCH 3/8] Cap the results cache's DuckDB memory at twice the budget The byte budget counts the results kept, not the working memory DuckDB needs to sort or profile them. With no memory limit of its own, the cache's DuckDB could use most of the machine's memory before failing, so the process could be killed first. It now gets a hard limit of twice the budget (at least 64 MB), so a sort that would need more fails with an error and leaves the view as it was. Co-Authored-By: Claude Opus 5.5 --- ggsql-jupyter/AGENTS.md | 5 +-- ggsql-jupyter/src/main.rs | 4 ++- ggsql-jupyter/src/results/mod.rs | 54 +++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ggsql-jupyter/AGENTS.md b/ggsql-jupyter/AGENTS.md index 1c2714760..a513bbb9a 100644 --- a/ggsql-jupyter/AGENTS.md +++ b/ggsql-jupyter/AGENTS.md @@ -188,10 +188,11 @@ So `RenderHints::canvas` picks a height (golden ratio, close to ggplot2's defaul `results/` keeps recent query results and answers a small, frontend-neutral set of calls on them (`methods.rs` documents each: `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`). Nothing in it knows about Positron: the ggsql extension translates Positron's Data Explorer protocol into these calls (see [`ggsql-vscode/AGENTS.md`](../ggsql-vscode/AGENTS.md)). The kernel's only Positron-specific part is the route: a `call_method` request on the `positron.ui` comm — what an extension's `session.callMethod(name, params)` sends — whose method starts with `result_` goes to `methods::call`, and its answer or error comes back as the JSON-RPC reply (errors with `plot::comm::RpcError`'s encoding). A `call_method` for any other name is an error, like an unknown plot-comm method, so a caller expecting something the kernel lacks fails visibly; any other UI request gets `result: null`. - **Which results are kept.** Only when the frontend asks, with `--keep-results` or `GGSQL_KEEP_RESULTS` — the extension sets it for a console session whose results it can open — so a session nobody explores holds nothing beyond what it displays. Then every tabular result of a non-silent execution is kept, and its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data. -- **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key) and a sort or filter that needs room fails with an error saying so. DuckDB's own working memory during a sort comes on top of the estimate. +- **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key) and a sort or filter that needs room fails with an error saying so. +- **The cache's DuckDB has a hard memory limit of twice the budget** (`memory_limit_for`, at least 64 MB). The budget counts what is kept, by Arrow's estimate; DuckDB holds a table less compactly than that and needs working memory on top to sort and profile, so the limit is what bounds the process. A budget too large to set as a limit leaves DuckDB's default, most of the machine's memory. - **Opening copies the result into the results cache.** A `CacheTables` (the caching reader's bookkeeping, `ggsql::reader::cache`) on its own in-memory DuckDB, which `ResultStore` creates itself. The session's reader is never wrapped, so the user's dialect is untouched. The copy adds `__ggsql_row__`, each row's position in the query's output, and drops the Arrow data, so a result is held once. It stays **pinned** while any view of it is open; once closed, it goes when the budget needs the room, and reopening before then reuses it. A result can have several views, each with its own filters and sort. - **Views end with the UI comm.** Closing it closes every view, since nothing could close them once it has gone; that is what releases the pins of a frontend that reloaded or crashed. -- **Nothing is written to disk.** The cache sets `temp_directory = ''`, so DuckDB cannot spill: a query that needs more memory than the limit fails with an error instead. +- **Nothing is written to disk.** The cache sets `temp_directory = ''`, so DuckDB cannot spill: a query that needs more memory than the limit fails with DuckDB's out-of-memory error, and the view it was for is left as it was. - **Sorting and filtering build an order table** of `(pos, __ggsql_row__)` per view (`view.rs`); a page joins it back to the result by `pos`, so scrolling never sorts again. Ties break on `__ggsql_row__`, so paging is stable and clearing the sort restores the query's own order. An unfiltered, unsorted view has none and pages by `__ggsql_row__` directly. A new sort or set of filters is built under a new table name and only then replaces the old one, so one that fails leaves the view as it was. - **A page is bounded.** At most `MAX_PAGE_ROWS`; a range is cut short at the end of the view, and an index outside it is an error, so a cell never lands in another row's place. Scattered indices fetch just those rows. - **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, booleans and regular expressions come from the dialect the SQL is for (`SqlDialect::sql_regex_match`, `None` where a backend has none), and `LIKE` escapes with `!`, which no backend reads as a string escape. diff --git a/ggsql-jupyter/src/main.rs b/ggsql-jupyter/src/main.rs index 62ded8fb2..791563b07 100644 --- a/ggsql-jupyter/src/main.rs +++ b/ggsql-jupyter/src/main.rs @@ -68,7 +68,9 @@ struct Args { /// their sorted or filtered views, by the size of their data. Closed /// results go first when room is needed, then the oldest unopened ones; /// open ones are never dropped, so a result that does not fit beside them - /// is not kept. + /// is not kept. Sorting and profiling them takes working memory on top, + /// so the results cache as a whole is limited to twice this; a sort that + /// would need more fails with an error. #[arg( long, env = "GGSQL_RESULTS_MAX_BYTES", diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs index 8b3c08d1b..846b135bf 100644 --- a/ggsql-jupyter/src/results/mod.rs +++ b/ggsql-jupyter/src/results/mod.rs @@ -18,6 +18,11 @@ //! unopened ones; an open view is never pulled from under its frontend, so //! when open views alone fill the budget, a new result is not kept and a sort //! or filter that needs room fails with an error saying so. +//! +//! The budget counts what is kept, not the working memory DuckDB needs to +//! sort or profile it, so the cache's DuckDB is also given a hard memory limit +//! of [`memory_limit_for`] the budget. A query that would pass it fails with +//! DuckDB's out-of-memory error, which reaches the frontend as an error. mod export; pub mod filter; @@ -42,6 +47,24 @@ pub const ROW: &str = "__ggsql_row__"; /// The default byte budget for everything the results API keeps. pub const DEFAULT_MAX_BYTES: u64 = 1024 * 1024 * 1024; +/// The least memory the cache's DuckDB is given, however small the budget. +const MIN_MEMORY_LIMIT: u64 = 64 * 1024 * 1024; + +/// The most memory limit set on the cache's DuckDB; a budget that would pass +/// it leaves DuckDB's own default, which is most of the machine's memory. +const MAX_MEMORY_LIMIT: u64 = 1 << 50; + +/// The memory the cache's DuckDB may use, kept tables and working memory +/// together, for a budget of `max_bytes`: twice the budget, since DuckDB holds +/// a table less compactly than Arrow's estimate of it and needs room on top to +/// sort and profile. `None` for no limit of ours. +pub fn memory_limit_for(max_bytes: u64) -> Option { + max_bytes + .checked_mul(2) + .map(|limit| limit.max(MIN_MEMORY_LIMIT)) + .filter(|&limit| limit <= MAX_MEMORY_LIMIT) +} + /// A result's column. #[derive(Debug, Clone, Serialize)] pub struct Column { @@ -99,12 +122,16 @@ pub struct ResultStore { } impl ResultStore { - /// A store keeping at most `max_bytes`, by Arrow's estimate of the data. + /// A store keeping at most `max_bytes`, by Arrow's estimate of the data, + /// whose DuckDB uses at most [`memory_limit_for`] it. pub fn new(max_bytes: u64) -> Result { let backend = ggsql::reader::connection::reader_from_uri("duckdb://memory")?; // An in-memory DuckDB may otherwise spill to temporary files under // memory pressure; results must never be written to disk. backend.execute_sql("SET temp_directory = ''")?; + if let Some(limit) = memory_limit_for(max_bytes) { + backend.execute_sql(&format!("SET memory_limit = '{}KiB'", limit / 1024))?; + } // The store evicts to its own budget, which spans more than the cache. let cache = CacheTables::new(backend, u64::MAX); Ok(Self { @@ -852,6 +879,28 @@ mod tests { assert_eq!(page(&mut store, &view, 0, 8, 0), before); } + #[test] + fn test_the_memory_limit_follows_the_budget() { + assert_eq!( + memory_limit_for(1024 * 1024 * 1024), + Some(2048 * 1024 * 1024) + ); + // However small the budget, DuckDB gets room to run. + assert_eq!(memory_limit_for(1), Some(MIN_MEMORY_LIMIT)); + // A budget too large to double, or to set, leaves DuckDB's default. + assert_eq!(memory_limit_for(u64::MAX), None); + assert_eq!(memory_limit_for(MAX_MEMORY_LIMIT), None); + + let store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let set = store + .cache + .backend() + .execute_sql("SELECT current_setting('memory_limit') AS m") + .unwrap(); + let set = ggsql::array_util::value_to_string(set.column("m").unwrap(), 0); + assert_eq!(set, "2.0 GiB"); + } + #[test] fn test_a_failed_filter_leaves_the_view_as_it_was() { let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); @@ -981,6 +1030,9 @@ mod tests { ) .unwrap_err(); assert!(err.to_lowercase().contains("memory"), "{err}"); + // The view is as it was, and still pages. + assert!(v.sort.is_empty()); + assert_eq!(page(&mut store, &view, 0, 1, 0), ["0", "1"]); } #[test] From 53114f3e1422615b8c0d4be823f5c8cceb101d82 Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Fri, 20 Nov 2026 10:48:50 -0600 Subject: [PATCH 4/8] Address second review of the results API - Make room only once a new result is known to fit beside open views, so an oversized insert drops nothing (`CacheTables::pinned_bytes`). - Treat a cell that does not parse cleanly as one statement in `result_sql`, rather than splitting around ERROR nodes. - Write `result_sql` through the dialect: string literals (`sql_string_literal`), LIKE (`sql_like`), NULL ordering (`sql_order_term`) and typed date/time literals. Co-Authored-By: Claude Opus 5.5 --- ggsql-jupyter/AGENTS.md | 6 +- ggsql-jupyter/src/results/filter.rs | 330 +++++++++++++++++++++++---- ggsql-jupyter/src/results/methods.rs | 2 +- ggsql-jupyter/src/results/mod.rs | 40 +++- src/AGENTS.md | 2 +- src/reader/cache.rs | 37 +++ src/reader/dialects/bigquery.rs | 9 + src/reader/dialects/clickhouse.rs | 9 + src/reader/dialects/databricks.rs | 4 + src/reader/dialects/mod.rs | 97 ++++++++ src/reader/dialects/mssql.rs | 11 + src/reader/dialects/mysql.rs | 9 + src/reader/dialects/redshift.rs | 4 + src/reader/dialects/snowflake.rs | 4 + src/reader/mod.rs | 32 +++ 15 files changed, 536 insertions(+), 60 deletions(-) diff --git a/ggsql-jupyter/AGENTS.md b/ggsql-jupyter/AGENTS.md index a513bbb9a..e00e0453c 100644 --- a/ggsql-jupyter/AGENTS.md +++ b/ggsql-jupyter/AGENTS.md @@ -188,16 +188,16 @@ So `RenderHints::canvas` picks a height (golden ratio, close to ggplot2's defaul `results/` keeps recent query results and answers a small, frontend-neutral set of calls on them (`methods.rs` documents each: `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`). Nothing in it knows about Positron: the ggsql extension translates Positron's Data Explorer protocol into these calls (see [`ggsql-vscode/AGENTS.md`](../ggsql-vscode/AGENTS.md)). The kernel's only Positron-specific part is the route: a `call_method` request on the `positron.ui` comm — what an extension's `session.callMethod(name, params)` sends — whose method starts with `result_` goes to `methods::call`, and its answer or error comes back as the JSON-RPC reply (errors with `plot::comm::RpcError`'s encoding). A `call_method` for any other name is an error, like an unknown plot-comm method, so a caller expecting something the kernel lacks fails visibly; any other UI request gets `result: null`. - **Which results are kept.** Only when the frontend asks, with `--keep-results` or `GGSQL_KEEP_RESULTS` — the extension sets it for a console session whose results it can open — so a session nobody explores holds nothing beyond what it displays. Then every tabular result of a non-silent execution is kept, and its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data. -- **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key) and a sort or filter that needs room fails with an error saying so. +- **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones — and only once it is known the new bytes fit beside what open views hold, so a result too big to keep drops nothing. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key) and a sort or filter that needs room fails with an error saying so. - **The cache's DuckDB has a hard memory limit of twice the budget** (`memory_limit_for`, at least 64 MB). The budget counts what is kept, by Arrow's estimate; DuckDB holds a table less compactly than that and needs working memory on top to sort and profile, so the limit is what bounds the process. A budget too large to set as a limit leaves DuckDB's default, most of the machine's memory. - **Opening copies the result into the results cache.** A `CacheTables` (the caching reader's bookkeeping, `ggsql::reader::cache`) on its own in-memory DuckDB, which `ResultStore` creates itself. The session's reader is never wrapped, so the user's dialect is untouched. The copy adds `__ggsql_row__`, each row's position in the query's output, and drops the Arrow data, so a result is held once. It stays **pinned** while any view of it is open; once closed, it goes when the budget needs the room, and reopening before then reuses it. A result can have several views, each with its own filters and sort. - **Views end with the UI comm.** Closing it closes every view, since nothing could close them once it has gone; that is what releases the pins of a frontend that reloaded or crashed. - **Nothing is written to disk.** The cache sets `temp_directory = ''`, so DuckDB cannot spill: a query that needs more memory than the limit fails with DuckDB's out-of-memory error, and the view it was for is left as it was. - **Sorting and filtering build an order table** of `(pos, __ggsql_row__)` per view (`view.rs`); a page joins it back to the result by `pos`, so scrolling never sorts again. Ties break on `__ggsql_row__`, so paging is stable and clearing the sort restores the query's own order. An unfiltered, unsorted view has none and pages by `__ggsql_row__` directly. A new sort or set of filters is built under a new table name and only then replaces the old one, so one that fails leaves the view as it was. - **A page is bounded.** At most `MAX_PAGE_ROWS`; a range is cut short at the end of the view, and an index outside it is an error, so a cell never lands in another row's place. Scattered indices fetch just those rows. -- **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, booleans and regular expressions come from the dialect the SQL is for (`SqlDialect::sql_regex_match`, `None` where a backend has none), and `LIKE` escapes with `!`, which no backend reads as a string escape. +- **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, a date, datetime or time as ISO 8601 without an offset (it becomes the dialect's typed literal, `sql_date_literal` and friends, since not every backend compares a string with a temporal column), anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, string and boolean literals, `LIKE` and regular expressions all come from the dialect the SQL is for (`SqlDialect::sql_string_literal`, `sql_like`, `sql_regex_match` — `None` where a backend has no regular expressions), and sort keys put NULLs last in either direction through `sql_order_term`, as the cache's DuckDB does, so exported SQL sorts as the view did, so the same filter is written correctly for a backend that reads backslashes in literals, has no `ESCAPE` clause, or has more `LIKE` wildcards. - **Kinds are DuckDB type names matched exactly** (`kind_of`): a column type that only contains `INT`, like `POINT`, is not an integer, and an `ENUM` is a string. -- **`result_sql` writes SQL for the session's database**, not the cache: the cell's statements before the last as they were, then the last as a derived table with the view's filters and sort, in the session's dialect. It is in the kernel because only the kernel knows that dialect. A filter it cannot express there, or one on a column whose name the query repeats (renamed `a_1` in the cache, but ambiguous in SQL), is left out with a `--` comment. +- **`result_sql` writes SQL for the session's database**, not the cache: the cell's statements before the last as they were (the whole cell is one statement when it does not parse cleanly), then the last as a derived table with the view's filters and sort, in the session's dialect. It is in the kernel because only the kernel knows that dialect. A filter it cannot express there, or one on a column whose name the query repeats (renamed `a_1` in the cache, but ambiguous in SQL), is left out with a `--` comment. `examples/results_bench.rs` measures it. At 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 3.4 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. diff --git a/ggsql-jupyter/src/results/filter.rs b/ggsql-jupyter/src/results/filter.rs index 34969266f..3efaa2487 100644 --- a/ggsql-jupyter/src/results/filter.rs +++ b/ggsql-jupyter/src/results/filter.rs @@ -4,10 +4,11 @@ //! [`SqlDialect`], with no database, so it is tested as plain string assembly. //! The same code writes SQL for the results cache, to filter and sort a view, //! and for the session's own database, for `result_sql`: identifiers, -//! booleans and regular expressions all come from the dialect it is given. +//! string and boolean literals, `LIKE` and regular expressions all come from +//! the dialect it is given. use super::Column; -use ggsql::naming::quote_literal; +use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use ggsql::parser::SourceTree; use ggsql::reader::SqlDialect; use serde::{Deserialize, Serialize}; @@ -156,8 +157,10 @@ pub fn predicate( } Filter::IsNull { negate: false, .. } => Ok(format!("{c} IS NULL")), Filter::IsNull { negate: true, .. } => Ok(format!("{c} IS NOT NULL")), - Filter::IsEmpty { negate: false, .. } => Ok(format!("{c} = ''")), - Filter::IsEmpty { negate: true, .. } => Ok(format!("{c} <> ''")), + Filter::IsEmpty { negate, .. } => { + let op = if *negate { "<>" } else { "=" }; + Ok(format!("{c} {op} {}", dialect.sql_string_literal(""))) + } Filter::IsTrue { .. } => Ok(format!("{c} = {}", dialect.sql_boolean_literal(true))), Filter::IsFalse { .. } => Ok(format!("{c} = {}", dialect.sql_boolean_literal(false))), Filter::Text { @@ -198,12 +201,54 @@ fn literal(value: &str, kind: &str, dialect: &dyn SqlDialect) -> Result Ok(dialect.sql_boolean_literal(false)), _ => Err(format!("{value:?} is not true or false")), }, - // Dates and times too: every backend compares a string literal against - // a temporal column by casting it. - _ => Ok(quote_literal(value)), + // A typed literal: not every backend casts a string to compare it with + // a temporal column (Trino refuses, Oracle reads it by a session + // setting). + "date" => parse_date(value.trim()) + .map(|d| dialect.sql_date_literal((d - epoch_date()).num_days() as i32)) + .ok_or_else(|| format!("{value:?} is not a date (YYYY-MM-DD)")), + "datetime" => parse_datetime(value.trim()) + .map(|dt| dialect.sql_datetime_literal(dt.and_utc().timestamp_micros())) + .ok_or_else(|| format!("{value:?} is not a date and time (YYYY-MM-DD HH:MM:SS)")), + "time" => parse_time(value.trim()) + .map(|t| { + let since_midnight = t - NaiveTime::MIN; + dialect.sql_time_literal(since_midnight.num_nanoseconds().unwrap_or(0)) + }) + .ok_or_else(|| format!("{value:?} is not a time (HH:MM:SS)")), + _ => Ok(dialect.sql_string_literal(value)), } } +fn epoch_date() -> NaiveDate { + NaiveDate::from_ymd_opt(1970, 1, 1).expect("the epoch is a date") +} + +fn parse_date(v: &str) -> Option { + NaiveDate::parse_from_str(v, "%Y-%m-%d").ok() +} + +/// An ISO date and time, with `T` or a space between them and optional +/// fractional seconds; a date alone is its midnight. No offset: a literal has +/// no time zone, so a database reads it in the session's own. +fn parse_datetime(v: &str) -> Option { + [ + "%Y-%m-%d %H:%M:%S%.f", + "%Y-%m-%dT%H:%M:%S%.f", + "%Y-%m-%d %H:%M", + "%Y-%m-%dT%H:%M", + ] + .iter() + .find_map(|f| NaiveDateTime::parse_from_str(v, f).ok()) + .or_else(|| parse_date(v).map(|d| d.and_time(NaiveTime::MIN))) +} + +fn parse_time(v: &str) -> Option { + ["%H:%M:%S%.f", "%H:%M"] + .iter() + .find_map(|f| NaiveTime::parse_from_str(v, f).ok()) +} + /// The predicate for a text filter. fn text( c: &str, @@ -214,7 +259,7 @@ fn text( ) -> Result { if how == TextMatch::Regex { return dialect - .sql_regex_match(c, "e_literal(term), case_sensitive) + .sql_regex_match(c, &dialect.sql_string_literal(term), case_sensitive) .ok_or_else(|| "This database does not support regular-expression search".into()); } @@ -226,36 +271,29 @@ fn text( TextMatch::EndsWith => (format!("%{escaped}"), false), TextMatch::Regex => unreachable!("handled above"), }; - let (lhs, rhs) = if case_sensitive { - (c.to_string(), quote_literal(&pattern)) + let like = dialect.sql_like(c, &pattern, case_sensitive); + Ok(if negate { + format!("NOT ({like})") } else { - ( - format!("LOWER({c})"), - format!("LOWER({})", quote_literal(&pattern)), - ) - }; - let not = if negate { "NOT " } else { "" }; - Ok(format!("{lhs} {not}LIKE {rhs} ESCAPE '{LIKE_ESCAPE}'")) + like + }) } -/// `LIKE`'s escape character. Not a backslash, which MySQL and Snowflake -/// read as an escape inside the string literal itself, so `'\'` would never -/// end there. -const LIKE_ESCAPE: char = '!'; - -/// Escape `LIKE`'s wildcards, so a search term matches literally. +/// Escape `LIKE`'s wildcards, and the backslash that escapes them, so a +/// search term matches literally under [`SqlDialect::sql_like`]. fn escape_like(term: &str) -> String { let mut out = String::with_capacity(term.len()); for ch in term.chars() { - if matches!(ch, '%' | '_') || ch == LIKE_ESCAPE { - out.push(LIKE_ESCAPE); + if matches!(ch, '%' | '_' | '\\') { + out.push('\\'); } out.push(ch); } out } -/// The sort keys as `ORDER BY` terms, without the keyword. +/// The sort keys as `ORDER BY` terms, without the keyword, with NULLs last +/// in either direction whatever the backend's default. pub fn order_terms( keys: &[SortKey], columns: &[Column], @@ -264,8 +302,9 @@ pub fn order_terms( keys.iter() .map(|key| { let column = column_of(columns, key.column)?; - let direction = if key.ascending { "ASC" } else { "DESC" }; - Ok(format!("{} {direction}", dialect.quote_ident(&column.name))) + // NULLs last either way, which is where Positron's Data + // Explorer shows them. + Ok(dialect.sql_order_term(&dialect.quote_ident(&column.name), key.ascending, true)) }) .collect() } @@ -336,13 +375,20 @@ pub fn to_sql( } /// A cell's statements: those before the last, and the last, which is the one -/// whose result was kept. A query that does not parse is one statement. +/// whose result was kept. A query that does not parse cleanly is one +/// statement. fn split_statements(query: &str) -> (Vec, String) { let whole = || (Vec::new(), query.trim().trim_end_matches(';').to_string()); let Ok(tree) = SourceTree::new(query) else { return whole(); }; let root = tree.root(); + // Tree-sitter recovers from what it cannot parse with ERROR nodes, and + // statement boundaries found around one are guesses, as in + // `SELECT $$x;y$$`. + if root.has_error() { + return whole(); + } let mut statements: Vec = tree .find_nodes(&root, "(sql_statement) @s") .into_iter() @@ -434,9 +480,9 @@ mod tests { #[test] fn test_text_escapes_wildcards_and_folds_case() { assert_eq!( - sql(json!({"type": "text", "column": 1, "match": "contains", "term": "50%_!"})) + sql(json!({"type": "text", "column": 1, "match": "contains", "term": "50%_\\"})) .unwrap(), - "LOWER(\"name\") LIKE LOWER('%50!%!_!!%') ESCAPE '!'" + r#"LOWER("name") LIKE LOWER('%50\%\_\\%') ESCAPE '\'"# ); assert_eq!( sql( @@ -444,7 +490,12 @@ mod tests { "case_sensitive": true}) ) .unwrap(), - "\"name\" LIKE 'a%' ESCAPE '!'" + r#""name" LIKE 'a%' ESCAPE '\'"# + ); + assert_eq!( + sql(json!({"type": "text", "column": 1, "match": "not_contains", "term": "a"})) + .unwrap(), + r#"NOT (LOWER("name") LIKE LOWER('%a%') ESCAPE '\')"# ); } @@ -452,13 +503,13 @@ mod tests { fn test_the_like_escape_matches_literally_in_duckdb() { let reader = ggsql::reader::connection::reader_from_uri("duckdb://memory").unwrap(); let filter: Filter = serde_json::from_value( - json!({"type": "text", "column": 1, "match": "contains", "term": "5%!_"}), + json!({"type": "text", "column": 1, "match": "contains", "term": "5%\\_"}), ) .unwrap(); let p = predicate(&filter, &columns(), &DuckDbDialect).unwrap(); let df = reader .execute_sql(&format!( - "SELECT name FROM (VALUES ('a5%!_b'), ('a5x!_b'), ('a5%!xb')) t(name) WHERE {p}" + r"SELECT name FROM (VALUES ('a5%\_b'), ('a5x\_b'), ('a5%\xb')) t(name) WHERE {p}" )) .unwrap(); assert_eq!(df.height(), 1); @@ -477,6 +528,89 @@ mod tests { assert!(predicate(&f, &columns(), &AnsiDialect).is_err()); } + fn temporal_columns() -> Vec { + [("d", "date"), ("ts", "datetime"), ("t", "time")] + .iter() + .map(|(name, kind)| Column { + name: name.to_string(), + type_name: String::new(), + kind: kind.to_string(), + repeated: false, + }) + .collect() + } + + #[test] + fn test_temporal_values_are_typed_literals_of_the_dialect() { + let f = |column: usize, value: &str, dialect: &dyn SqlDialect| { + let filter: Filter = serde_json::from_value( + json!({"type": "compare", "column": column, "op": ">=", "value": value}), + ) + .unwrap(); + predicate(&filter, &temporal_columns(), dialect) + }; + assert_eq!( + f(0, "2024-03-05", &AnsiDialect).unwrap(), + "\"d\" >= DATE '2024-03-05'" + ); + assert_eq!( + f(1, "2024-03-05T06:07:08.5", &AnsiDialect).unwrap(), + "\"ts\" >= TIMESTAMP '2024-03-05 06:07:08.500000'" + ); + assert_eq!( + f(1, "2024-03-05", &AnsiDialect).unwrap(), + "\"ts\" >= TIMESTAMP '2024-03-05 00:00:00'" + ); + assert_eq!( + f(2, "06:07", &AnsiDialect).unwrap(), + "\"t\" >= TIME '06:07:00'" + ); + // An override, not a string the backend may refuse to compare. + let trino = dialect_for_scheme("trino").unwrap(); + let mssql = dialect_for_scheme("mssql").unwrap(); + assert_eq!( + f(0, "2024-03-05", trino).unwrap(), + format!("\"d\" >= {}", trino.sql_date_literal(19787)) + ); + assert_ne!( + f(0, "2024-03-05", mssql).unwrap(), + f(0, "2024-03-05", &AnsiDialect).unwrap() + ); + for (column, bad) in [ + (0, "March 5"), + (1, "2024-03-05 06:07:08+02:00"), + (2, "noon"), + ] { + assert!(f(column, bad, &AnsiDialect).is_err(), "{bad}"); + } + } + + #[test] + fn test_temporal_filters_run_in_duckdb() { + let reader = ggsql::reader::connection::reader_from_uri("duckdb://memory").unwrap(); + let filters: Vec = serde_json::from_value(json!([ + {"type": "between", "column": 0, "low": "2024-01-01", "high": "2024-01-31"}, + {"type": "compare", "column": 1, "op": ">", "value": "2024-01-10 12:00"}, + {"type": "in", "column": 2, "values": ["09:30:00"]} + ])) + .unwrap(); + let where_ = filters + .iter() + .map(|f| predicate(f, &temporal_columns(), &DuckDbDialect).unwrap()) + .collect::>() + .join(" AND "); + let df = reader + .execute_sql(&format!( + "SELECT * FROM (VALUES \ + (DATE '2024-01-15', TIMESTAMP '2024-01-15 00:00:00', TIME '09:30:00'), \ + (DATE '2024-01-15', TIMESTAMP '2024-01-10 11:00:00', TIME '09:30:00'), \ + (DATE '2024-02-15', TIMESTAMP '2024-02-15 00:00:00', TIME '09:30:00')\ + ) t(d, ts, t) WHERE {where_}" + )) + .unwrap(); + assert_eq!(df.height(), 1); + } + #[test] fn test_in() { assert_eq!( @@ -498,6 +632,53 @@ mod tests { assert!(sql(json!({"type": "compare", "column": 2, "op": "=", "value": "maybe"})).is_err()); } + #[test] + fn test_sort_puts_nulls_last_in_every_dialect() { + let desc = [SortKey { + column: 0, + ascending: false, + }]; + let mysql = dialect_for_scheme("mysql").unwrap(); + assert_eq!( + order_terms(&desc, &columns(), mysql).unwrap(), + ["CASE WHEN `n` IS NULL THEN 1 ELSE 0 END, `n` DESC"] + ); + assert_eq!( + order_terms(&desc, &columns(), &AnsiDialect).unwrap(), + ["\"n\" DESC NULLS LAST"] + ); + // And both shapes run, with the NULL last either way. + for uri in ["duckdb://memory", "sqlite://memory"] { + let reader = ggsql::reader::connection::reader_from_uri(uri).unwrap(); + let dialect = reader.dialect(); + for ascending in [true, false] { + let key = [SortKey { + column: 0, + ascending, + }]; + let terms = order_terms(&key, &columns(), dialect).unwrap().join(", "); + let both = [terms, nulls_order_by(dialect, ascending)]; + for order in both { + let df = reader + .execute_sql(&format!( + "SELECT n FROM (SELECT NULL AS n UNION ALL SELECT 1 UNION ALL SELECT 2) t \ + ORDER BY {order}" + )) + .unwrap(); + assert!(df.column("n").unwrap().is_null(2), "{uri} {order}"); + } + } + } + } + + /// The `CASE` form MySQL and SQL Server get, for running elsewhere. + fn nulls_order_by(dialect: &dyn SqlDialect, ascending: bool) -> String { + let mysql = dialect_for_scheme("mysql").unwrap(); + mysql + .sql_order_term("n", ascending, true) + .replace('`', &dialect.quote_ident("")[..1]) + } + #[test] fn test_an_unknown_column_is_an_error() { assert!(sql(json!({"type": "is_null", "column": 9})).is_err()); @@ -543,26 +724,73 @@ mod tests { ") AS \"result\"", "WHERE \"n\" > 2", " AND \"name\" IS NULL", - "ORDER BY \"n\" ASC", + "ORDER BY \"n\" ASC NULLS LAST", ] ); } #[test] - fn test_to_sql_quotes_for_the_session_dialect() { - let filters = filters(json!([ - {"type": "text", "column": 1, "match": "contains", "term": "x"} + fn test_to_sql_writes_the_session_dialect() { + let both = filters(json!([ + {"type": "text", "column": 1, "match": "contains", "term": "it's"}, + {"type": "compare", "column": 1, "op": "=", "value": "a\\b"} ])); - for scheme in ["mysql", "bigquery", "clickhouse", "databricks"] { + let cases = [ + ( + "mysql", + r"LOWER(`name`) LIKE LOWER('%it\'s%') ESCAPE '\\'", + r"`name` = 'a\\b'", + ), + ( + "databricks", + r"LOWER(`name`) LIKE LOWER('%it\'s%') ESCAPE '\\'", + r"`name` = 'a\\b'", + ), + ( + "bigquery", + r"LOWER(`name`) LIKE LOWER('%it\'s%')", + r"`name` = 'a\\b'", + ), + ( + "clickhouse", + r"LOWER(`name`) LIKE LOWER('%it\'s%')", + r"`name` = 'a\\b'", + ), + ( + "snowflake", + r#"LOWER("name") LIKE LOWER('%it\'s%') ESCAPE '\\'"#, + r#""name" = 'a\\b'"#, + ), + ( + "postgres", + r#"LOWER("name") LIKE LOWER('%it''s%') ESCAPE '\'"#, + r#""name" = 'a\b'"#, + ), + ]; + for (scheme, like, compare) in cases { let dialect = dialect_for_scheme(scheme).unwrap(); - let sql = to_sql(&filters, &BY_N, "SELECT 1", &columns(), dialect).join("\n"); - assert!( - sql.contains("LOWER(`name`) LIKE LOWER('%x%') ESCAPE '!'"), - "{scheme}: {sql}" - ); - assert!(sql.contains("ORDER BY `n` ASC"), "{scheme}: {sql}"); - assert!(!sql.contains("\"name\""), "{scheme}: {sql}"); + let sql = to_sql(&both, &BY_N, "SELECT 1", &columns(), dialect).join("\n"); + assert!(sql.contains(&format!("WHERE {like}\n")), "{scheme}: {sql}"); + assert!(sql.contains(&format!("AND {compare}\n")), "{scheme}: {sql}"); } + // The wildcard tail of a backslash-escaping dialect's pattern still + // ends the literal: `'%\\%'`, not `'%\%'`. + let backslash = filters(json!([ + {"type": "text", "column": 1, "match": "ends_with", "term": "\\", "case_sensitive": true} + ])); + let mysql = dialect_for_scheme("mysql").unwrap(); + let sql = to_sql(&backslash, &[], "SELECT 1", &columns(), mysql).join("\n"); + assert!(sql.contains(r"`name` LIKE '%\\\\' ESCAPE '\\'"), "{sql}"); + } + + #[test] + fn test_to_sql_escapes_a_tsql_character_class() { + let filters = filters(json!([ + {"type": "text", "column": 1, "match": "starts_with", "term": "[a]", "case_sensitive": true} + ])); + let mssql = dialect_for_scheme("mssql").unwrap(); + let sql = to_sql(&filters, &[], "SELECT 1", &columns(), mssql).join("\n"); + assert!(sql.contains(r#""name" LIKE '\[a]%' ESCAPE '\'"#), "{sql}"); } #[test] @@ -577,7 +805,7 @@ mod tests { "FROM (", " SELECT * FROM t", ") AS \"result\"", - "ORDER BY \"n\" ASC", + "ORDER BY \"n\" ASC NULLS LAST", ] ); // And what it writes runs. @@ -590,6 +818,14 @@ mod tests { assert_eq!(last.unwrap().height(), 2); } + #[test] + fn test_a_query_that_does_not_parse_cleanly_is_one_statement() { + // Parsed as `SELECT 1`, an ERROR node, then `x;y$$ AS s` as a + // statement of its own. + let query = "SELECT 1; SELECT $$x;y$$ AS s"; + assert_eq!(split_statements(query), (Vec::new(), query.to_string())); + } + #[test] fn test_to_sql_leaves_out_a_repeated_column() { let mut cols = columns(); diff --git a/ggsql-jupyter/src/results/methods.rs b/ggsql-jupyter/src/results/methods.rs index ae19319b9..3bf43c425 100644 --- a/ggsql-jupyter/src/results/methods.rs +++ b/ggsql-jupyter/src/results/methods.rs @@ -219,7 +219,7 @@ mod tests { " SELECT * FROM t", ") AS \"result\"", "WHERE \"n\" > 1", - "ORDER BY \"n\" DESC" + "ORDER BY \"n\" DESC NULLS LAST" ] ); diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs index 846b135bf..58f1e3a0a 100644 --- a/ggsql-jupyter/src/results/mod.rs +++ b/ggsql-jupyter/src/results/mod.rs @@ -151,15 +151,17 @@ impl ResultStore { } /// Drop closed results, then the oldest unopened ones, until `extra` more - /// bytes fit in the budget. Returns whether they do. + /// bytes fit in the budget. Returns whether they do. When they cannot fit + /// even with everything droppable gone, nothing is dropped. fn make_room(&mut self, extra: u64) -> Result { - let limit = self.max_bytes.saturating_sub(extra); - let outside_cache: u64 = self.pending.iter().map(|p| p.bytes).sum::() - + self - .views - .values() - .map(ResultView::order_bytes) - .sum::(); + let orders: u64 = self.views.values().map(ResultView::order_bytes).sum(); + // Open views' results and order tables stay whatever is dropped. + let kept = self.cache.pinned_bytes()? + orders; + if kept.saturating_add(extra) > self.max_bytes { + return Ok(false); + } + let limit = self.max_bytes - extra; + let outside_cache: u64 = self.pending.iter().map(|p| p.bytes).sum::() + orders; self.cache.evict_to(limit.saturating_sub(outside_cache))?; self.forget_evicted()?; while self.used()? > limit && self.pending.pop_front().is_some() {} @@ -782,6 +784,28 @@ mod tests { assert!(store.open(&b).is_ok()); } + #[test] + fn test_a_result_that_cannot_fit_beside_open_views_drops_nothing() { + let one = bytes_of(&numbers(100)); + let mut store = ResultStore::new(4 * one).unwrap(); + let open = keep(&mut store, numbers(100), "open".into(), String::new()); + let closed = keep(&mut store, numbers(100), "closed".into(), String::new()); + let pending = keep(&mut store, numbers(100), "pending".into(), String::new()); + store.open(&open).unwrap(); + let v = store.open(&closed).unwrap().view; + store.close(&v).unwrap(); + + // Under the budget, but not beside the open result. + let big = numbers(350); + assert!(bytes_of(&big) <= 4 * one); + assert!(store + .insert(big, "big".into(), String::new()) + .unwrap() + .is_none()); + assert!(store.open(&closed).is_ok()); + assert!(store.open(&pending).is_ok()); + } + #[test] fn test_the_oldest_unopened_result_is_dropped_to_make_room() { let one = bytes_of(&numbers(10)); diff --git a/src/AGENTS.md b/src/AGENTS.md index ea367145a..052bded6f 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -57,7 +57,7 @@ Grammar lives in [`/tree-sitter-ggsql/`](../tree-sitter-ggsql/) — when adding | `spec.rs` | `ResolvedSpec` (`ResolvedPlot` | `ResolvedTable`) returned by `execute()`, plus DataFrame conversion | — | | `data.rs` | Bundled sample datasets — the `ggsql:` builtins | `builtin-data` | -`SqlDialect` trait in `mod.rs` lets each driver supply its own type names, information-schema queries, and spatial helper methods (`sql_st_transform`, `sql_geometry_to_wkb`, `sql_geometry_bbox`, `sql_ensure_geometry`, `sql_select_replace`, `sql_spatial_setup`). +`SqlDialect` trait in `mod.rs` lets each driver supply its own type names, information-schema queries, literal and pattern syntax (`quote_ident`, `sql_string_literal`, `sql_like`, `sql_regex_match`, `sql_order_term`), and spatial helper methods (`sql_st_transform`, `sql_geometry_to_wkb`, `sql_geometry_bbox`, `sql_ensure_geometry`, `sql_select_replace`, `sql_spatial_setup`). **Caching layer.** `CachingReader` (`cache.rs`) wraps a primary reader plus an in-memory `CacheBackend`, splitting work across two `Reader` surfaces. **`execute_sql` = source**: base reads of the user's data plus user setup/DML run on the primary (with result memoization), except `ggsql:` builtins, the `__ggsql_cache_meta__` table, and reads that reference a cache-resident internal table, which go to the cache. **`execute_sql_cached` = compute**: all dialect-generated/derived SQL (schema probes, stats, projection/map transforms, spatial setup, final layer queries — everything operating on `__ggsql_*` tables) runs on the cache; it defaults to `execute_sql` so a plain reader runs everything on one connection. Cache routing is by **exact-identifier membership** in the set of tables registered into the cache. Memoization keys on `hash(primary_uri + sql)` and is tracked in the `__ggsql_cache_meta__` table inside the cache backend. Each memoized read is bounded by a **TTL** (default 300s) and the whole memo by an **LRU byte budget** (default 512 MB); both are configurable via `CacheConfig` (env `GGSQL_CACHE_DISABLED`/`GGSQL_CACHE_TTL`/`GGSQL_CACHE_MAX_BYTES`, or per-connection URI query parameters `?cache_ttl=…&cache_max_bytes=…&cache_disabled=…`). The `__ggsql_cache_meta__` table is queryable for introspection (`SELECT * FROM __ggsql_cache_meta__`). Pure/non-visual SQL (CLI table fallback, Jupyter) goes through `execute_sql` so it reads the primary rather than the empty cache. `Reader::materialize_table` (default = `CREATE TEMP TABLE` on the reader, no Rust roundtrip) is overridden to read the body via the source surface and `register()` the result into the cache, so the primary is never written to; `Reader::caches_sources()` (default `false`, `true` for `CachingReader`) gates the executor's per-layer source staging: file sources are staged on the cache surface, while identifiers go through `materialize_table`, which routes the read to the cache (CTEs, builtins, cache-resident tables) or the primary as needed. `dialect()` returns the **cache** dialect, and every compute-surface failure is prefixed with the cache backend's scheme (``on the `duckdb` cache backend: …``) so a cache-dialect driver error is not mistaken for one from the user's own connection. Selected via the composite `+://` scheme (`reader_from_uri` / `split_cache_uri`) or the CLI `--cache` flag; off by default. The memo's bookkeeping — registering a table, its row in `__ggsql_cache_meta__`, LRU eviction under the byte budget — is `CacheTables`, which other callers use on a backend of their own (the Jupyter kernel's results API does, without wrapping the user's reader). An entry can be **pinned**, exempting it from eviction until unpinned (pins are kept in memory, not in the metadata table); `CachingReader` never pins, and the TTL and enabled checks stay in `CachingReader`. `CacheTables` takes a byte budget rather than a `CacheConfig`, and `evict_to`/`total_bytes` let a caller with a wider budget evict to that instead. diff --git a/src/reader/cache.rs b/src/reader/cache.rs index aef3d03fe..c12e4394f 100644 --- a/src/reader/cache.rs +++ b/src/reader/cache.rs @@ -333,6 +333,29 @@ impl CacheTables { Ok(total.max(0) as u64) } + /// The pinned entries' total byte estimate: what no eviction can free. + pub fn pinned_bytes(&self) -> Result { + self.ensure_meta_table()?; + let pinned = self.pinned.borrow(); + if pinned.is_empty() { + return Ok(0); + } + let keys: Vec = pinned.iter().map(|k| naming::quote_literal(k)).collect(); + drop(pinned); + let df = self.backend.execute_sql(&format!( + "SELECT CAST(COALESCE(SUM(byte_estimate), 0) AS BIGINT) AS n FROM {} \ + WHERE cache_key IN ({})", + naming::quote_ident(naming::CACHE_META_TABLE), + keys.join(", ") + ))?; + let total = if df.height() == 0 { + 0 + } else { + as_i64(df.column("n")?)?.value(0) + }; + Ok(total.max(0) as u64) + } + /// Evict unpinned entries, least recently used first, until the total /// byte estimate is within `max_bytes` or nothing unpinned is left. A /// caller with a budget wider than this cache's passes what it allows. @@ -1444,6 +1467,20 @@ mod behavior_tests { assert_eq!(rows.height(), 3); } + #[test] + fn test_pinned_bytes_counts_only_pinned_entries() { + let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), u64::MAX); + assert_eq!(tables.pinned_bytes().unwrap(), 0); + let frame = || df! { "x" => vec![1_i64, 2, 3] }.unwrap(); + tables.put("a", "a", "__t_a__", frame(), true).unwrap(); + tables.put("b", "b", "__t_b__", frame(), false).unwrap(); + let total = tables.total_bytes().unwrap(); + assert!(total > 0); + assert_eq!(tables.pinned_bytes().unwrap() * 2, total); + tables.set_pinned("a", false).unwrap(); + assert_eq!(tables.pinned_bytes().unwrap(), 0); + } + #[test] fn test_unpinning_makes_an_entry_evictable() { let tables = tiny_budget_tables(); diff --git a/src/reader/dialects/bigquery.rs b/src/reader/dialects/bigquery.rs index 8b1b45a3a..dc221c0df 100644 --- a/src/reader/dialects/bigquery.rs +++ b/src/reader/dialects/bigquery.rs @@ -16,6 +16,15 @@ use crate::reader::SqlDialect; pub struct BigQueryDialect; impl SqlDialect for BigQueryDialect { + fn sql_string_literal(&self, value: &str) -> String { + super::backslash_quote_literal(value) + } + + // No `ESCAPE` clause; a backslash always escapes in `LIKE`. + fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { + super::like_without_escape_clause(self, expr, pattern, case_sensitive) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("FLOAT64"), diff --git a/src/reader/dialects/clickhouse.rs b/src/reader/dialects/clickhouse.rs index 53f2ec50d..463ec78e9 100644 --- a/src/reader/dialects/clickhouse.rs +++ b/src/reader/dialects/clickhouse.rs @@ -30,6 +30,15 @@ fn float_args(exprs: &[&str]) -> String { } impl SqlDialect for ClickHouseDialect { + fn sql_string_literal(&self, value: &str) -> String { + super::backslash_quote_literal(value) + } + + // No `ESCAPE` clause; a backslash always escapes in `LIKE`. + fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { + super::like_without_escape_clause(self, expr, pattern, case_sensitive) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("Nullable(Float64)"), diff --git a/src/reader/dialects/databricks.rs b/src/reader/dialects/databricks.rs index 9dcfa9704..478e61506 100644 --- a/src/reader/dialects/databricks.rs +++ b/src/reader/dialects/databricks.rs @@ -11,6 +11,10 @@ use crate::reader::SqlDialect; pub struct DatabricksDialect; impl SqlDialect for DatabricksDialect { + fn sql_string_literal(&self, value: &str) -> String { + super::backslash_quote_literal(value) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("DOUBLE"), diff --git a/src/reader/dialects/mod.rs b/src/reader/dialects/mod.rs index b5f4c7a67..11ae4a236 100644 --- a/src/reader/dialects/mod.rs +++ b/src/reader/dialects/mod.rs @@ -56,6 +56,55 @@ pub(crate) fn backtick_quote_ident(name: &str) -> String { format!("`{}`", name.replace('`', "``")) } +/// String literal for backends that read a backslash inside a string literal +/// as an escape (MySQL/MariaDB, ClickHouse, BigQuery, Databricks, Snowflake, +/// Redshift): backslashes and single quotes are both backslash-escaped. +/// BigQuery and Databricks have no `''` escape at all. +pub(crate) fn backslash_quote_literal(value: &str) -> String { + format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) +} + +/// [`SqlDialect::sql_like`] with `ESCAPE` naming the backslash, as a string +/// literal of the dialect's own, so the clause is valid wherever its literals +/// are. +pub(crate) fn like_with_escape_clause( + dialect: &D, + expr: &str, + pattern: &str, + case_sensitive: bool, +) -> String { + format!( + "{} ESCAPE {}", + like_without_escape_clause(dialect, expr, pattern, case_sensitive), + dialect.sql_string_literal("\\") + ) +} + +/// [`SqlDialect::sql_like`] for backends whose `LIKE` has no `ESCAPE` clause +/// and always escapes with a backslash (ClickHouse, BigQuery). +pub(crate) fn like_without_escape_clause( + dialect: &D, + expr: &str, + pattern: &str, + case_sensitive: bool, +) -> String { + let pattern = dialect.sql_string_literal(pattern); + if case_sensitive { + format!("{expr} LIKE {pattern}") + } else { + format!("LOWER({expr}) LIKE LOWER({pattern})") + } +} + +/// [`SqlDialect::sql_order_term`] for backends without `NULLS FIRST` / +/// `NULLS LAST` (MySQL/MariaDB, SQL Server): a leading key that sorts the +/// NULLs to the chosen end, then the expression itself. +pub(crate) fn nulls_order_by_case(expr: &str, ascending: bool, nulls_last: bool) -> String { + let (null, other) = if nulls_last { (1, 0) } else { (0, 1) }; + let direction = if ascending { "ASC" } else { "DESC" }; + format!("CASE WHEN {expr} IS NULL THEN {null} ELSE {other} END, {expr} {direction}") +} + /// `1`/`0` boolean literal for backends without a real BOOLEAN literal /// (SQL Server, Oracle, SQLite). pub(crate) fn one_zero_boolean_literal(value: bool) -> String { @@ -150,6 +199,54 @@ mod tests { assert_eq!(d.type_names().number, expected); } + #[test] + fn string_literals_follow_the_backends_escapes() { + let value = r"it's a\b"; + for scheme in ["mysql", "bigquery", "clickhouse", "databricks", "snowflake"] { + let d = dialect_for_scheme(scheme).unwrap(); + assert_eq!(d.sql_string_literal(value), r"'it\'s a\\b'", "{scheme}"); + } + for scheme in ["duckdb", "postgres", "sqlite", "mssql", "trino"] { + let d = dialect_for_scheme(scheme).unwrap(); + assert_eq!(d.sql_string_literal(value), r"'it''s a\b'", "{scheme}"); + } + } + + #[test] + fn like_names_its_escape_where_the_backend_has_the_clause() { + let like = |scheme: &str| { + dialect_for_scheme(scheme) + .unwrap() + .sql_like("c", r"a\%", true) + }; + assert_eq!(like("duckdb"), r"c LIKE 'a\%' ESCAPE '\'"); + assert_eq!(like("mysql"), r"c LIKE 'a\\%' ESCAPE '\\'"); + assert_eq!(like("bigquery"), r"c LIKE 'a\\%'"); + assert_eq!(like("clickhouse"), r"c LIKE 'a\\%'"); + assert_eq!( + dialect_for_scheme("mssql") + .unwrap() + .sql_like("c", "[x]%", false), + r"LOWER(c) LIKE LOWER('\[x]%') ESCAPE '\'" + ); + } + + /// The default `LIKE` escape runs, with the backslash as written. + #[cfg(feature = "duckdb")] + #[test] + fn like_escape_runs_in_duckdb() { + use crate::reader::Reader; + let reader = + crate::reader::DuckDBReader::from_connection_string("duckdb://memory").unwrap(); + let p = DuckDbDialect.sql_like("s", r"%a\_b\\%", true); + let df = reader + .execute_sql(&format!( + r"SELECT s FROM (VALUES ('xa_b\y'), ('xaxb\y'), ('xa_by')) t(s) WHERE {p}" + )) + .unwrap(); + assert_eq!(df.height(), 1); + } + /// Evaluate the CASE emulations against DuckDB's native /// `GREATEST`/`LEAST` so the left-folded chain is checked as executed /// SQL, not just as a string. diff --git a/src/reader/dialects/mssql.rs b/src/reader/dialects/mssql.rs index 4fa0fdd37..78188c610 100644 --- a/src/reader/dialects/mssql.rs +++ b/src/reader/dialects/mssql.rs @@ -18,6 +18,17 @@ use crate::reader::SqlDialect; pub struct MssqlDialect; impl SqlDialect for MssqlDialect { + // No `NULLS FIRST` / `NULLS LAST`. + fn sql_order_term(&self, expr: &str, ascending: bool, nulls_last: bool) -> String { + super::nulls_order_by_case(expr, ascending, nulls_last) + } + + // `[` opens a character class in T-SQL's `LIKE`, so it is escaped too. + fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { + let pattern = pattern.replace('[', "\\["); + super::like_with_escape_clause(self, expr, &pattern, case_sensitive) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("FLOAT"), diff --git a/src/reader/dialects/mysql.rs b/src/reader/dialects/mysql.rs index 95fb4ab48..12129044d 100644 --- a/src/reader/dialects/mysql.rs +++ b/src/reader/dialects/mysql.rs @@ -10,6 +10,15 @@ use crate::reader::SqlDialect; pub struct MySqlDialect; impl SqlDialect for MySqlDialect { + // No `NULLS FIRST` / `NULLS LAST`. + fn sql_order_term(&self, expr: &str, ascending: bool, nulls_last: bool) -> String { + super::nulls_order_by_case(expr, ascending, nulls_last) + } + + fn sql_string_literal(&self, value: &str) -> String { + super::backslash_quote_literal(value) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("DOUBLE"), diff --git a/src/reader/dialects/redshift.rs b/src/reader/dialects/redshift.rs index 71e86d5c5..4c0e3a9bc 100644 --- a/src/reader/dialects/redshift.rs +++ b/src/reader/dialects/redshift.rs @@ -17,6 +17,10 @@ use crate::reader::SqlDialect; pub struct RedshiftDialect; impl SqlDialect for RedshiftDialect { + fn sql_string_literal(&self, value: &str) -> String { + super::backslash_quote_literal(value) + } + fn sql_temporal_as_number( &self, expr: &str, diff --git a/src/reader/dialects/snowflake.rs b/src/reader/dialects/snowflake.rs index da5d4bc1e..c3235d1e2 100644 --- a/src/reader/dialects/snowflake.rs +++ b/src/reader/dialects/snowflake.rs @@ -11,6 +11,10 @@ use crate::reader::SqlDialect; pub struct SnowflakeDialect; impl SqlDialect for SnowflakeDialect { + fn sql_string_literal(&self, value: &str) -> String { + super::backslash_quote_literal(value) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("DOUBLE"), diff --git a/src/reader/mod.rs b/src/reader/mod.rs index 469f722bc..43bc8f3ac 100644 --- a/src/reader/mod.rs +++ b/src/reader/mod.rs @@ -450,6 +450,15 @@ pub trait SqlDialect { } } + /// SQL string literal for `value`. + /// + /// Default is SQL-standard single quotes with embedded quotes doubled. + /// Override for backends that read a backslash inside a string literal as + /// an escape (`dialects::backslash_quote_literal`). + fn sql_string_literal(&self, value: &str) -> String { + naming::quote_literal(value) + } + /// Whether `expr` matches the regular expression in the SQL string /// literal `pattern`, or `None` when the backend has no regular /// expressions. Default `None`. @@ -458,6 +467,29 @@ pub trait SqlDialect { None } + /// Whether `expr` matches the `LIKE` pattern `pattern`, given as raw text + /// rather than a SQL literal: `%` and `_` are wildcards, and a backslash + /// makes the next character literal. Case-insensitive matching folds both + /// sides with `LOWER`. + /// + /// Default names the backslash with an `ESCAPE` clause. Override for + /// backends with no `ESCAPE` clause, or with more wildcards than `%` and + /// `_`. + fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { + dialects::like_with_escape_clause(self, expr, pattern, case_sensitive) + } + + /// One `ORDER BY` term sorting `expr` with its NULLs placed explicitly, + /// since backends disagree on where they go by default. + /// + /// Default uses `NULLS FIRST` / `NULLS LAST`. Override for backends + /// without them (`dialects::nulls_order_by_case`). + fn sql_order_term(&self, expr: &str, ascending: bool, nulls_last: bool) -> String { + let direction = if ascending { "ASC" } else { "DESC" }; + let nulls = if nulls_last { "LAST" } else { "FIRST" }; + format!("{expr} {direction} NULLS {nulls}") + } + // ===================================================================== // Statement generators and DDL // ===================================================================== From 0c32f6d5cdf914c6a7949fcd9a0d061da70afcff Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Fri, 20 Nov 2026 12:27:56 -0600 Subject: [PATCH 5/8] Address pre-merge review of the results API - Close every view when a new UI comm opens, so a frontend that reloaded without closing its old comm releases its pins. - Cap exports at 100,000 rows, separately from the page limit. - Write timezone-aware timestamps as timestamps in UTC. - Reserve budget for the cached copy's row column, and for every order-table rebuild, including clearing a sort or filters. - Leave an already open Data Explorer's view alone when showing it fails, and forget closed datasets with their session. - Regex search for ten more dialects; leave a non-query last statement unwrapped in `result_sql`; create the results store on first use. - Expect `NULLS LAST` in the integration test. Co-Authored-By: Claude Opus 5.5 --- ggsql-jupyter/AGENTS.md | 12 +- ggsql-jupyter/src/display.rs | 45 ++++++-- ggsql-jupyter/src/kernel.rs | 48 ++++++-- ggsql-jupyter/src/results/filter.rs | 94 ++++++++++++++-- ggsql-jupyter/src/results/methods.rs | 6 +- ggsql-jupyter/src/results/mod.rs | 127 ++++++++++++++++++++-- ggsql-jupyter/src/results/view.rs | 55 ++++++++-- ggsql-jupyter/tests/test_integration.py | 34 ++++-- ggsql-vscode/src/dataExplorer/handler.ts | 10 ++ ggsql-vscode/src/resultLinks.ts | 4 +- ggsql-vscode/src/test/resultLinks.test.ts | 18 +++ src/reader/dialects/bigquery.rs | 6 + src/reader/dialects/clickhouse.rs | 6 + src/reader/dialects/databricks.rs | 6 + src/reader/dialects/datafusion.rs | 6 + src/reader/dialects/druid.rs | 6 + src/reader/dialects/mod.rs | 49 ++++++++- src/reader/dialects/mysql.rs | 6 + src/reader/dialects/oracle.rs | 6 + src/reader/dialects/redshift.rs | 6 + src/reader/dialects/snowflake.rs | 8 ++ src/reader/dialects/trino.rs | 6 + 22 files changed, 496 insertions(+), 68 deletions(-) diff --git a/ggsql-jupyter/AGENTS.md b/ggsql-jupyter/AGENTS.md index e00e0453c..8cbb36207 100644 --- a/ggsql-jupyter/AGENTS.md +++ b/ggsql-jupyter/AGENTS.md @@ -187,17 +187,17 @@ So `RenderHints::canvas` picks a height (golden ratio, close to ggplot2's defaul `results/` keeps recent query results and answers a small, frontend-neutral set of calls on them (`methods.rs` documents each: `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`). Nothing in it knows about Positron: the ggsql extension translates Positron's Data Explorer protocol into these calls (see [`ggsql-vscode/AGENTS.md`](../ggsql-vscode/AGENTS.md)). The kernel's only Positron-specific part is the route: a `call_method` request on the `positron.ui` comm — what an extension's `session.callMethod(name, params)` sends — whose method starts with `result_` goes to `methods::call`, and its answer or error comes back as the JSON-RPC reply (errors with `plot::comm::RpcError`'s encoding). A `call_method` for any other name is an error, like an unknown plot-comm method, so a caller expecting something the kernel lacks fails visibly; any other UI request gets `result: null`. -- **Which results are kept.** Only when the frontend asks, with `--keep-results` or `GGSQL_KEEP_RESULTS` — the extension sets it for a console session whose results it can open — so a session nobody explores holds nothing beyond what it displays. Then every tabular result of a non-silent execution is kept, and its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data. -- **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones — and only once it is known the new bytes fit beside what open views hold, so a result too big to keep drops nothing. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key) and a sort or filter that needs room fails with an error saying so. +- **Which results are kept.** Only when the frontend asks, with `--keep-results` or `GGSQL_KEEP_RESULTS` — the extension sets it for a console session whose results it can open — so a session nobody explores holds nothing beyond what it displays. Then every tabular result of a non-silent execution is kept, and its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data. The store and its DuckDB are created on first use, so a session that keeps nothing opens no database for them. +- **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones — and only once it is known the new bytes fit beside what open views hold, so a result too big to keep drops nothing. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key), and opening a result or a sort or filter that needs room fails with an error saying so. Opening reserves the copy's row column as well as the result, and any rebuild of an order table reserves a new one, including clearing the sort of a filtered view or the filters of a sorted one. - **The cache's DuckDB has a hard memory limit of twice the budget** (`memory_limit_for`, at least 64 MB). The budget counts what is kept, by Arrow's estimate; DuckDB holds a table less compactly than that and needs working memory on top to sort and profile, so the limit is what bounds the process. A budget too large to set as a limit leaves DuckDB's default, most of the machine's memory. - **Opening copies the result into the results cache.** A `CacheTables` (the caching reader's bookkeeping, `ggsql::reader::cache`) on its own in-memory DuckDB, which `ResultStore` creates itself. The session's reader is never wrapped, so the user's dialect is untouched. The copy adds `__ggsql_row__`, each row's position in the query's output, and drops the Arrow data, so a result is held once. It stays **pinned** while any view of it is open; once closed, it goes when the budget needs the room, and reopening before then reuses it. A result can have several views, each with its own filters and sort. -- **Views end with the UI comm.** Closing it closes every view, since nothing could close them once it has gone; that is what releases the pins of a frontend that reloaded or crashed. +- **Views end with the UI comm.** Closing it closes every view, since nothing could close them once it has gone, and so does opening a new one: a frontend that reloaded or crashed opens a new UI comm without closing the old. That is what releases its pins. - **Nothing is written to disk.** The cache sets `temp_directory = ''`, so DuckDB cannot spill: a query that needs more memory than the limit fails with DuckDB's out-of-memory error, and the view it was for is left as it was. - **Sorting and filtering build an order table** of `(pos, __ggsql_row__)` per view (`view.rs`); a page joins it back to the result by `pos`, so scrolling never sorts again. Ties break on `__ggsql_row__`, so paging is stable and clearing the sort restores the query's own order. An unfiltered, unsorted view has none and pages by `__ggsql_row__` directly. A new sort or set of filters is built under a new table name and only then replaces the old one, so one that fails leaves the view as it was. -- **A page is bounded.** At most `MAX_PAGE_ROWS`; a range is cut short at the end of the view, and an index outside it is an error, so a cell never lands in another row's place. Scattered indices fetch just those rows. -- **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, a date, datetime or time as ISO 8601 without an offset (it becomes the dialect's typed literal, `sql_date_literal` and friends, since not every backend compares a string with a temporal column), anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, string and boolean literals, `LIKE` and regular expressions all come from the dialect the SQL is for (`SqlDialect::sql_string_literal`, `sql_like`, `sql_regex_match` — `None` where a backend has no regular expressions), and sort keys put NULLs last in either direction through `sql_order_term`, as the cache's DuckDB does, so exported SQL sorts as the view did, so the same filter is written correctly for a backend that reads backslashes in literals, has no `ESCAPE` clause, or has more `LIKE` wildcards. +- **A page is bounded, and so is an export.** A page holds at most `MAX_PAGE_ROWS`, and an export at most `MAX_EXPORT_ROWS` (100,000), since it is built as text on the message loop, outside DuckDB's memory limit; copying a whole column of a longer result is an error that says so. For both, a range is cut short at the end of the view, and an index outside it is an error, so a cell never lands in another row's place. Scattered indices fetch just those rows. +- **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, a date, datetime or time as ISO 8601 without an offset (it becomes the dialect's typed literal, `sql_date_literal` and friends, since not every backend compares a string with a temporal column), anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, string and boolean literals, `LIKE` and regular expressions all come from the dialect the SQL is for (`SqlDialect::sql_string_literal`, `sql_like`, `sql_regex_match` — `None` where a backend has no regular expressions, as SQLite and SQL Server), so the same filter is written correctly for a backend that reads backslashes in literals, has no `ESCAPE` clause, or has more `LIKE` wildcards. Sort keys put NULLs last in either direction through `sql_order_term`, as the cache's DuckDB does, so exported SQL sorts as the view did. - **Kinds are DuckDB type names matched exactly** (`kind_of`): a column type that only contains `INT`, like `POINT`, is not an integer, and an `ENUM` is a string. -- **`result_sql` writes SQL for the session's database**, not the cache: the cell's statements before the last as they were (the whole cell is one statement when it does not parse cleanly), then the last as a derived table with the view's filters and sort, in the session's dialect. It is in the kernel because only the kernel knows that dialect. A filter it cannot express there, or one on a column whose name the query repeats (renamed `a_1` in the cache, but ambiguous in SQL), is left out with a `--` comment. +- **`result_sql` writes SQL for the session's database**, not the cache: the cell's statements before the last as they were (the whole cell is one statement when it does not parse cleanly), then the last as a derived table with the view's filters and sort, in the session's dialect. A last statement that is not a query (`DESCRIBE`, `SHOW`, …) cannot be a derived table, so it is written as it was, with a comment saying the filters and sort were left out. It is in the kernel because only the kernel knows that dialect. A filter it cannot express there, or one on a column whose name the query repeats (renamed `a_1` in the cache, but ambiguous in SQL), is left out with a `--` comment. `examples/results_bench.rs` measures it. At 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 3.4 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. diff --git a/ggsql-jupyter/src/display.rs b/ggsql-jupyter/src/display.rs index 85ddb2fdb..1877f4e3b 100644 --- a/ggsql-jupyter/src/display.rs +++ b/ggsql-jupyter/src/display.rs @@ -309,18 +309,32 @@ fn html_rows_shown(height: usize, hints: &RenderHints) -> usize { } /// A cell as a person reads it: `value_to_string`, except that dates are -/// written as dates rather than as the day or millisecond counts Arrow stores. +/// written as dates rather than as the day or millisecond counts Arrow stores, +/// and timestamps with a time zone as timestamps in UTC with their offset. pub(crate) fn cell_text(col: &arrow::array::ArrayRef, row: usize) -> String { + use arrow::array::Array; use arrow::datatypes::DataType; use arrow::util::display::{ArrayFormatter, FormatOptions}; - match col.data_type() { - DataType::Date32 | DataType::Date64 if !arrow::array::Array::is_null(col, row) => { - ArrayFormatter::try_new(col.as_ref(), &FormatOptions::default()) - .map(|f| f.value(row).to_string()) - .unwrap_or_else(|_| ggsql::array_util::value_to_string(col, row)) - } - _ => ggsql::array_util::value_to_string(col, row), - } + let formatted = |cell: &dyn Array| { + ArrayFormatter::try_new(cell, &FormatOptions::default()) + .map(|f| f.value(0).to_string()) + .ok() + }; + let text = match col.data_type() { + _ if col.is_null(row) => None, + DataType::Date32 | DataType::Date64 => formatted(&col.slice(row, 1)), + // Arrow here reads only fixed offsets as zones, not names such as + // the `UTC` DuckDB tags these with. The stored value is an instant, + // so retagging it as `+00:00` writes the same moment. + DataType::Timestamp(unit, Some(_)) => arrow::compute::cast( + &col.slice(row, 1), + &DataType::Timestamp(*unit, Some("+00:00".into())), + ) + .ok() + .and_then(|cell| formatted(&cell)), + _ => None, + }; + text.unwrap_or_else(|| ggsql::array_util::value_to_string(col, row)) } /// Convert DataFrame to an HTML table of its first `row_limit` rows @@ -409,6 +423,19 @@ mod tests { assert!(dataframe_to_text(&df, 1).contains("2024-01-01")); } + #[test] + fn test_timestamps_with_a_time_zone_are_written_as_timestamps() { + let df = ggsql::reader::connection::reader_from_uri("duckdb://memory") + .unwrap() + .execute_sql( + "SELECT TIMESTAMPTZ '2024-01-02 03:04:05+00' AS t, NULL::TIMESTAMPTZ AS missing", + ) + .unwrap(); + let text = cell_text(df.column("t").unwrap(), 0); + assert_eq!(text, "2024-01-02T03:04:05Z"); + assert_eq!(cell_text(df.column("missing").unwrap(), 0), "null"); + } + #[test] fn test_a_console_gets_a_comm_even_without_an_adapter() { // No GPU here, and the console still opens a comm — Positron would diff --git a/ggsql-jupyter/src/kernel.rs b/ggsql-jupyter/src/kernel.rs index a88c7eed9..60af719ec 100644 --- a/ggsql-jupyter/src/kernel.rs +++ b/ggsql-jupyter/src/kernel.rs @@ -66,8 +66,11 @@ pub struct KernelServer { ui_comm_id: Option, connection_comm_id: Option, data_explorer_comms: HashMap, - /// Query results kept for the results API. - results: ResultStore, + /// Query results kept for the results API, created when first needed so + /// a session that keeps none opens no database for them. + results: Option, + /// `--results-max-bytes`: the results store's budget. + results_max_bytes: u64, /// `--keep-results`: whether to keep results for the results API, which /// the frontend that will explore them turns on. keep_results: bool, @@ -152,7 +155,8 @@ impl KernelServer { ui_comm_id: None, connection_comm_id: None, data_explorer_comms: HashMap::new(), - results: ResultStore::new(results_max_bytes)?, + results: None, + results_max_bytes, keep_results, console_rows, }; @@ -430,7 +434,10 @@ impl KernelServer { if let Ok(ExecutionResult::DataFrame(df)) = &executed { if self.keep_results && !silent && df.width() > 0 { let title = format!("Query [{}]", self.execution_count); - match self.results.insert(df.clone(), title, query_of(code)) { + let kept = self + .results() + .and_then(|store| store.insert(df.clone(), title, query_of(code))); + match kept { Ok(key) => retained = key, Err(e) => tracing::warn!("Keeping a result failed: {}", e), } @@ -627,6 +634,12 @@ impl KernelServer { } "positron.ui" => { tracing::info!("Registering positron.ui comm: {}", comm_id); + // A frontend that reloaded or crashed opens a new UI comm + // without closing the old one, and the views opened over that + // could never be closed. + if let Some(store) = self.results.as_mut() { + store.close_all(); + } self.ui_comm_id = Some(comm_id.to_string()); } _ => { @@ -760,10 +773,17 @@ impl KernelServer { let name = data["params"]["method"].as_str().unwrap_or(""); let args = &data["params"]["params"]; let answer = if methods::handles(name) { - let session = methods::Session { - reader: self.executor.reader(), - }; - methods::call(&mut self.results, &session, name, args) + match self.results() { + Ok(_) => { + let session = methods::Session { + reader: self.executor.reader(), + }; + let store = + self.results.as_mut().expect("created just above"); + methods::call(store, &session, name, args) + } + Err(e) => Err(format!("The results store failed: {e}")), + } } else { Err(format!("Unknown method {name:?}")) }; @@ -928,7 +948,9 @@ impl KernelServer { self.ui_comm_id = None; // The views were opened over this comm, and nothing can close // them once it has gone. - self.results.close_all(); + if let Some(store) = self.results.as_mut() { + store.close_all(); + } } else if self.plot_comms.remove(comm_id).is_some() { tracing::info!("Closing positron.plot comm {}", comm_id); self.plot_order.retain(|id| id != comm_id); @@ -1537,6 +1559,14 @@ impl KernelServer { Ok(()) } + /// The results store, created on first use. + fn results(&mut self) -> Result<&mut ResultStore> { + if self.results.is_none() { + self.results = Some(ResultStore::new(self.results_max_bytes)?); + } + Ok(self.results.as_mut().expect("created just above")) + } + /// Send a status message async fn send_status(&mut self, state: &str, parent: &JupyterMessage) -> Result<()> { self.send_iopub("status", json!({"execution_state": state}), parent) diff --git a/ggsql-jupyter/src/results/filter.rs b/ggsql-jupyter/src/results/filter.rs index 3efaa2487..7a27406d4 100644 --- a/ggsql-jupyter/src/results/filter.rs +++ b/ggsql-jupyter/src/results/filter.rs @@ -260,7 +260,9 @@ fn text( if how == TextMatch::Regex { return dialect .sql_regex_match(c, &dialect.sql_string_literal(term), case_sensitive) - .ok_or_else(|| "This database does not support regular-expression search".into()); + .ok_or_else(|| { + "ggsql cannot write a regular-expression search for this database".into() + }); } let escaped = escape_like(term); @@ -352,10 +354,26 @@ pub fn to_sql( } } - let (setup, last) = split_statements(query); + let Statements { + setup, + last, + last_is_query, + } = split_statements(query); for stmt in setup { lines.extend(format!("{stmt};").lines().map(str::to_string)); } + if !last_is_query { + // Only a query can be wrapped, so the rest is said rather than written. + if !predicates.is_empty() || !keys.is_empty() { + lines.push( + "-- Filters and sort left out: the last statement is not a query \ + that can be filtered or sorted" + .to_string(), + ); + } + lines.extend(format!("{last};").lines().map(str::to_string)); + return lines; + } lines.push("SELECT *".to_string()); lines.extend( format!("FROM {}", subquery(&last, dialect)) @@ -374,11 +392,23 @@ pub fn to_sql( lines } -/// A cell's statements: those before the last, and the last, which is the one -/// whose result was kept. A query that does not parse cleanly is one -/// statement. -fn split_statements(query: &str) -> (Vec, String) { - let whole = || (Vec::new(), query.trim().trim_end_matches(';').to_string()); +/// A cell's last statement, whose result was kept, and those before it. +struct Statements { + setup: Vec, + last: String, + /// Whether the last statement is a query that can be a derived table. + /// `DESCRIBE`, `SHOW`, `PRAGMA` and the like are not. + last_is_query: bool, +} + +/// A cell's statements. A query that does not parse cleanly is one statement, +/// taken to be a query. +fn split_statements(query: &str) -> Statements { + let whole = || Statements { + setup: Vec::new(), + last: query.trim().trim_end_matches(';').to_string(), + last_is_query: true, + }; let Ok(tree) = SourceTree::new(query) else { return whole(); }; @@ -389,7 +419,7 @@ fn split_statements(query: &str) -> (Vec, String) { if root.has_error() { return whole(); } - let mut statements: Vec = tree + let mut statements: Vec<(String, bool)> = tree .find_nodes(&root, "(sql_statement) @s") .into_iter() // The outermost statements: a subquery can be a statement of its own. @@ -403,11 +433,23 @@ fn split_statements(query: &str) -> (Vec, String) { } true }) - .map(|node| tree.get_text(&node).trim().to_string()) - .filter(|text| !text.is_empty()) + .map(|node| { + let is_query = node.child(0).is_some_and(|c| { + matches!( + c.kind(), + "select_statement" | "with_statement" | "from_statement" + ) + }); + (tree.get_text(&node).trim().to_string(), is_query) + }) + .filter(|(text, _)| !text.is_empty()) .collect(); match statements.pop() { - Some(last) => (statements, last), + Some((last, last_is_query)) => Statements { + setup: statements.into_iter().map(|(text, _)| text).collect(), + last, + last_is_query, + }, None => whole(), } } @@ -823,7 +865,35 @@ mod tests { // Parsed as `SELECT 1`, an ERROR node, then `x;y$$ AS s` as a // statement of its own. let query = "SELECT 1; SELECT $$x;y$$ AS s"; - assert_eq!(split_statements(query), (Vec::new(), query.to_string())); + let split = split_statements(query); + assert!(split.setup.is_empty()); + assert_eq!(split.last, query); + assert!(split.last_is_query); + } + + #[test] + fn test_to_sql_does_not_wrap_a_statement_that_is_not_a_query() { + let lines = to_sql( + &[], + &BY_N, + "CREATE TABLE t AS SELECT 1 AS n;\nDESCRIBE t", + &columns(), + &AnsiDialect, + ); + assert_eq!( + lines, + [ + "CREATE TABLE t AS SELECT 1 AS n;", + "-- Filters and sort left out: the last statement is not a query that can be \ + filtered or sorted", + "DESCRIBE t;", + ] + ); + // Queries are wrapped, however they start. + for query in ["WITH a AS (SELECT 1 AS n) SELECT * FROM a", "FROM t"] { + let lines = to_sql(&[], &BY_N, query, &columns(), &AnsiDialect); + assert_eq!(lines[0], "SELECT *", "{query}: {lines:?}"); + } } #[test] diff --git a/ggsql-jupyter/src/results/methods.rs b/ggsql-jupyter/src/results/methods.rs index 3bf43c425..b8229c3e3 100644 --- a/ggsql-jupyter/src/results/methods.rs +++ b/ggsql-jupyter/src/results/methods.rs @@ -16,14 +16,16 @@ //! A sort or set of filters that cannot be applied at all is an error, and //! leaves the view as it was. //! -//! A page holds at most [`MAX_PAGE_ROWS`] rows. A range is cut short at the end -//! of the view; an index outside it is an error. +//! A page holds at most [`MAX_PAGE_ROWS`] rows, and an export (every row of +//! the view when `rows` is left out) at most [`MAX_EXPORT_ROWS`]. A range is +//! cut short at the end of the view; an index outside it is an error. //! //! `result_sql` is here rather than in a frontend because only the kernel //! knows the session's database: it writes the view as SQL for that dialect, //! which a frontend could not do from the neutral calls alone. //! //! [`MAX_PAGE_ROWS`]: super::view::MAX_PAGE_ROWS +//! [`MAX_EXPORT_ROWS`]: super::view::MAX_EXPORT_ROWS use super::filter::{Filter, SortKey}; use super::view::{ProfileRequest, Rows}; diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs index 58f1e3a0a..f42070a64 100644 --- a/ggsql-jupyter/src/results/mod.rs +++ b/ggsql-jupyter/src/results/mod.rs @@ -80,6 +80,9 @@ pub struct Column { pub repeated: bool, } +/// Bytes the cached copy of a result adds per row: its `__ggsql_row__`. +const ROW_BYTES: u64 = 8; + /// A result not yet opened, held as its Arrow data. struct Pending { key: String, @@ -221,6 +224,24 @@ impl ResultStore { pub fn open(&mut self, key: &str) -> Result { if let Some(i) = self.pending.iter().position(|p| p.key == key) { let pending = self.pending.remove(i).expect("position is in range"); + // Out of the queue, so making room cannot drop it; its copy is + // budgeted afresh, with the row column the copy adds. + let need = pending.bytes + pending.df.height() as u64 * ROW_BYTES; + let fits = match self.make_room(need) { + Ok(fits) => fits, + Err(e) => { + self.pending.insert(i, pending); + return Err(e); + } + }; + if !fits { + self.pending.insert(i, pending); + bail!( + "Opening this result needs more memory than the data explorer's budget \ + has left ({} bytes). Close another Data Explorer tab and try again.", + self.max_bytes + ); + } match self.copy_into_cache(&pending) { Ok(opened) => { self.opened.insert(pending.key, opened); @@ -354,7 +375,14 @@ impl ResultStore { /// Set a view's sort keys, within the budget. pub fn set_sort(&mut self, view: &str, keys: Vec) -> Result<(), String> { - if !keys.is_empty() { + // An order table is rebuilt whenever the view keeps one: clearing + // the sort of a filtered view rebuilds it too. + let filtered = !self + .view(view) + .map_err(|e| e.to_string())? + .filters + .is_empty(); + if !keys.is_empty() || filtered { self.reserve_order(view).map_err(|e| e.to_string())?; } let (v, cache) = self.view_mut(view).map_err(|e| e.to_string())?; @@ -367,7 +395,10 @@ impl ResultStore { view: &str, filters: Vec, ) -> Result>, String> { - if !filters.is_empty() { + // As for a sort: clearing the filters of a sorted view rebuilds its + // order table too. + let sorted = !self.view(view).map_err(|e| e.to_string())?.sort.is_empty(); + if !filters.is_empty() || sorted { self.reserve_order(view).map_err(|e| e.to_string())?; } let (v, cache) = self.view_mut(view).map_err(|e| e.to_string())?; @@ -755,11 +786,15 @@ mod tests { estimate_bytes(df) as u64 } + /// What opening `numbers(n)` reserves: the result and its row column. + fn opened_bytes(n: usize) -> u64 { + bytes_of(&numbers(n)) + n as u64 * ROW_BYTES + } + #[test] fn test_an_open_result_survives_the_budget_and_a_closed_one_does_not() { - // Room for two results and an order table, not three results. - let one = bytes_of(&numbers(100)); - let mut store = ResultStore::new(2 * one + 100 * 16).unwrap(); + // Room for two opened results and an order table, not three results. + let mut store = ResultStore::new(2 * opened_bytes(100) + 100 * 16).unwrap(); let a = keep(&mut store, numbers(100), "a".into(), String::new()); let b = keep(&mut store, numbers(100), "b".into(), String::new()); let va = store.open(&a).unwrap().view; @@ -828,9 +863,8 @@ mod tests { #[test] fn test_order_tables_count_against_the_budget() { - let one = bytes_of(&numbers(1000)); - // Room for the result and less than one order table. - let mut store = ResultStore::new(one + 1000).unwrap(); + // Room for the opened result and less than one order table. + let mut store = ResultStore::new(opened_bytes(1000) + 1000).unwrap(); let key = keep(&mut store, numbers(1000), "a".into(), String::new()); let view = store.open(&key).unwrap().view; let err = store @@ -847,10 +881,52 @@ mod tests { assert_eq!(page(&mut store, &view, 0, 1, 0), ["0", "1"]); } + #[test] + fn test_clearing_the_filters_of_a_sorted_view_counts_against_the_budget() { + let order = 1000 * view::ORDER_ROW_BYTES; + // Room for both results, the filtered view's order table of 500 rows, + // and not quite a full one beside it. + let mut store = + ResultStore::new(2 * opened_bytes(1000) + order / 2 + order - 1000).unwrap(); + let a = keep(&mut store, numbers(1000), "a".into(), String::new()); + let va = store.open(&a).unwrap().view; + store + .set_sort( + &va, + vec![SortKey { + column: 0, + ascending: false, + }], + ) + .unwrap(); + let filters: Vec = serde_json::from_value( + serde_json::json!([{"type": "compare", "column": 0, "op": "<", "value": "500"}]), + ) + .unwrap(); + store.set_filters(&va, filters).unwrap(); + let b = keep(&mut store, numbers(1000), "b".into(), String::new()); + store.open(&b).unwrap(); + + // Unfiltered but still sorted, the view needs a new order table. + let err = store.set_filters(&va, Vec::new()).unwrap_err(); + assert!(err.contains("memory"), "{err}"); + assert_eq!(store.view(&va).unwrap().selected_rows, 500); + } + + #[test] + fn test_opening_counts_the_row_column() { + // Room for the result as kept, not for its copy with a row column. + let mut store = ResultStore::new(bytes_of(&numbers(1000))).unwrap(); + let key = keep(&mut store, numbers(1000), "a".into(), String::new()); + let err = store.open(&key).unwrap_err().to_string(); + assert!(err.contains("memory"), "{err}"); + // Still kept, so a larger budget's error would say the same again. + assert!(store.pending.iter().any(|p| p.key == key)); + } + #[test] fn test_close_all_unpins_every_result() { - let one = bytes_of(&numbers(100)); - let mut store = ResultStore::new(one).unwrap(); + let mut store = ResultStore::new(opened_bytes(100)).unwrap(); let a = keep(&mut store, numbers(100), "a".into(), String::new()); store.open(&a).unwrap(); store.open(&a).unwrap(); @@ -1001,6 +1077,37 @@ mod tests { .contains("at most")); } + #[test] + fn test_exports_are_bounded_more_loosely_than_pages() { + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep( + &mut store, + numbers(view::MAX_EXPORT_ROWS + 1), + "q".into(), + String::new(), + ); + let id = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&id).unwrap(); + // More rows than a page, as a selection may be. + let rows = Rows::Range { + first: 0, + last: view::MAX_PAGE_ROWS, + }; + let csv = v.export(cache.backend(), Some(&rows), &[0], "csv").unwrap(); + assert_eq!(csv.lines().count(), view::MAX_PAGE_ROWS + 2); + // A whole column of a longer result is refused, not built. + let err = v.export(cache.backend(), None, &[0], "csv").unwrap_err(); + assert!(err.contains("copied or exported"), "{err}"); + let all = Rows::Range { + first: 0, + last: usize::MAX, + }; + let err = v + .export(cache.backend(), Some(&all), &[0], "csv") + .unwrap_err(); + assert!(err.contains("copied or exported"), "{err}"); + } + #[test] fn test_running_out_of_memory_is_an_error_not_a_spill() { let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); diff --git a/ggsql-jupyter/src/results/view.rs b/ggsql-jupyter/src/results/view.rs index 26de3a871..8cf56d3e6 100644 --- a/ggsql-jupyter/src/results/view.rs +++ b/ggsql-jupyter/src/results/view.rs @@ -23,6 +23,38 @@ use std::collections::HashMap; /// The most rows one page may ask for. The Data Explorer asks for a screenful. pub const MAX_PAGE_ROWS: usize = 10_000; +/// The most rows one export may hold. An export is built as text on the +/// kernel's message loop, outside the cache's memory limit, so it is bounded +/// as a page is, only more loosely. +pub const MAX_EXPORT_ROWS: usize = 100_000; + +/// What a fetch is for, which sets its row limit and the wording of the error +/// when it is passed. +#[derive(Clone, Copy)] +enum Purpose { + Page, + Export, +} + +impl Purpose { + fn limit(self) -> usize { + match self { + Purpose::Page => MAX_PAGE_ROWS, + Purpose::Export => MAX_EXPORT_ROWS, + } + } + + fn too_many(self, n: usize) -> String { + match self { + Purpose::Page => format!("A page can hold at most {MAX_PAGE_ROWS} rows, not {n}"), + Purpose::Export => format!( + "At most {MAX_EXPORT_ROWS} rows can be copied or exported at once, not {n}. \ + Select fewer rows, or use Convert to Code to run the query yourself." + ), + } + } +} + /// Bytes an order table holds per row: two 64-bit integers. pub const ORDER_ROW_BYTES: u64 = 16; @@ -321,6 +353,7 @@ impl ResultView { reader: &dyn Reader, cols: &[usize], rows: &Rows, + purpose: Purpose, ) -> Result<(DataFrame, Vec), String> { let n = self.selected_rows; match rows { @@ -329,16 +362,16 @@ impl ResultView { if n == 0 || first > &last { return Ok((DataFrame::empty(), Vec::new())); } - if last - first + 1 > MAX_PAGE_ROWS { - return Err(too_many(last - first + 1)); + if last - first + 1 > purpose.limit() { + return Err(purpose.too_many(last - first + 1)); } let df = self.fetch(reader, cols, Fetch::Range(*first, last))?; let picked = (0..df.height()).collect(); Ok((df, picked)) } Rows::Indices { indices } => { - if indices.len() > MAX_PAGE_ROWS { - return Err(too_many(indices.len())); + if indices.len() > purpose.limit() { + return Err(purpose.too_many(indices.len())); } if let Some(bad) = indices.iter().find(|&&i| i >= n) { return Err(format!( @@ -382,7 +415,7 @@ impl ResultView { rows: &Rows, cols: &[usize], ) -> Result>, String> { - let (df, picked) = self.fetch_rows(reader, cols, rows)?; + let (df, picked) = self.fetch_rows(reader, cols, rows, Purpose::Page)?; if picked.is_empty() { return Ok(vec![Vec::new(); cols.len()]); } @@ -394,7 +427,8 @@ impl ResultView { } /// The cells at `rows` (every row when `None`) of `cols` as CSV, TSV or - /// HTML, with a header row of column names. + /// HTML, with a header row of column names. At most [`MAX_EXPORT_ROWS`] + /// rows. pub fn export( &self, reader: &dyn Reader, @@ -404,11 +438,14 @@ impl ResultView { ) -> Result { let (df, picked) = match rows { None => { + if self.selected_rows > MAX_EXPORT_ROWS { + return Err(Purpose::Export.too_many(self.selected_rows)); + } let df = self.fetch(reader, cols, Fetch::All)?; let n = df.height(); (df, (0..n).collect()) } - Some(rows) => self.fetch_rows(reader, cols, rows)?, + Some(rows) => self.fetch_rows(reader, cols, rows, Purpose::Export)?, }; let cells: Vec> = picked .iter() @@ -717,10 +754,6 @@ impl ResultView { } } -fn too_many(n: usize) -> String { - format!("A page can hold at most {MAX_PAGE_ROWS} rows, not {n}") -} - /// `SELECT COUNT(*)` over `table`, filtered by `predicate` when given. pub fn count_rows( reader: &dyn Reader, diff --git a/ggsql-jupyter/tests/test_integration.py b/ggsql-jupyter/tests/test_integration.py index b8dd9ccb0..4db7da852 100644 --- a/ggsql-jupyter/tests/test_integration.py +++ b/ggsql-jupyter/tests/test_integration.py @@ -530,27 +530,29 @@ def _result_key(self, client): assert not [m for m in messages if m["msg_type"] == "stream"] return results[0]["content"]["metadata"]["ggsql"]["result"] - def _open_ui(self, client): + def _open_ui(self, client, comm_id=None): client.session.send( client.shell_channel.socket, "comm_open", - {"comm_id": self.UI, "target_name": "positron.ui", "data": {}}, + {"comm_id": comm_id or self.UI, "target_name": "positron.ui", "data": {}}, ) drain_iopub(client) - def _close_ui(self, client): + def _close_ui(self, client, comm_id=None): client.session.send( - client.shell_channel.socket, "comm_close", {"comm_id": self.UI, "data": {}} + client.shell_channel.socket, + "comm_close", + {"comm_id": comm_id or self.UI, "data": {}}, ) drain_iopub(client) - def _call(self, client, method, params): + def _call(self, client, method, params, comm_id=None): """`callMethod(method, params)`: the reply's JSON-RPC data.""" client.session.send( client.shell_channel.socket, "comm_msg", { - "comm_id": self.UI, + "comm_id": comm_id or self.UI, "data": { "jsonrpc": "2.0", "id": f"{method}-1", @@ -597,7 +599,7 @@ def test_open_filter_sort_page_and_close(self, console_client): sql = self._call(console_client, "result_sql", {"view": view})["result"]["sql"] assert f" {self.QUERY}" in sql - assert 'ORDER BY "n" DESC' in sql + assert 'ORDER BY "n" DESC NULLS LAST' in sql assert self._call(console_client, "result_close", {"view": view})["result"] == {} # The cached copy is still there to reopen. @@ -712,6 +714,24 @@ def test_closing_the_ui_comm_closes_its_views(self, console_client): finally: self._close_ui(console_client) + def test_a_new_ui_comm_closes_the_views_of_the_old(self, console_client): + # A frontend that reloaded opens a new UI comm without closing its old. + key = self._result_key(console_client) + self._open_ui(console_client) + view = self._call(console_client, "result_open", {"key": key})["result"]["view"] + replacement = "test-ui-comm-2" + self._open_ui(console_client, replacement) + try: + data = self._call( + console_client, + "result_page", + {"view": view, "rows": {"first": 0, "last": 0}, "columns": [0]}, + replacement, + ) + assert "no view" in data["error"]["message"], data + finally: + self._close_ui(console_client, replacement) + class TestConsoleRows: """`GGSQL_CONSOLE_ROWS`, which the extension sets from `ggsql.consoleRows`.""" diff --git a/ggsql-vscode/src/dataExplorer/handler.ts b/ggsql-vscode/src/dataExplorer/handler.ts index 468bd0249..207c4bd9b 100644 --- a/ggsql-vscode/src/dataExplorer/handler.ts +++ b/ggsql-vscode/src/dataExplorer/handler.ts @@ -186,6 +186,11 @@ export class ResultExplorer { } this.down.delete(sessionId); this.forgetViews(sessionId); + for (const id of this.closed) { + if (parseDatasetId(id)?.sessionId === sessionId) { + this.closed.delete(id); + } + } }, }; } @@ -231,6 +236,11 @@ export class ResultExplorer { return this.open(id); } + /** Whether a Data Explorer is open on the dataset `id`. */ + isShown(id: string): boolean { + return this.shown.has(id); + } + /** The dataset `id`, opening a kernel view of it the first time. */ open(id: string): Promise { if (this.closed.has(id)) { diff --git a/ggsql-vscode/src/resultLinks.ts b/ggsql-vscode/src/resultLinks.ts index c54119af4..11e82f4cf 100644 --- a/ggsql-vscode/src/resultLinks.ts +++ b/ggsql-vscode/src/resultLinks.ts @@ -79,6 +79,8 @@ async function openResult( return; } const id = datasetId(sessionId, key); + // A Data Explorer already showing it owns its view, whatever happens here. + const alreadyShown = explorer.isShown(id); let opened = false; try { // Opened here rather than on the Data Explorer's first request, for @@ -91,7 +93,7 @@ async function openResult( displayName: dataset.result.title, }); } catch (err) { - if (opened) { + if (opened && !alreadyShown) { // No Data Explorer will close the view, so release it here. explorer.closeDataset(id); } diff --git a/ggsql-vscode/src/test/resultLinks.test.ts b/ggsql-vscode/src/test/resultLinks.test.ts index 05d271193..336aaeb43 100644 --- a/ggsql-vscode/src/test/resultLinks.test.ts +++ b/ggsql-vscode/src/test/resultLinks.test.ts @@ -163,6 +163,24 @@ suite('withResultLinks', () => { wrapped.dispose(); }); + test('a failed show leaves an already open Data Explorer\'s view alone', async () => { + const fake = fakeSession('console'); + let fail = false; + const wrapped = withResultLinks(fake.session, new ResultExplorer(() => { }), async () => { + if (fail) { + throw new Error('already open'); + } + }); + await wrapped.openResource!('x-ggsql-result:ggsql-abc/0f9e'); + await tick(); + fail = true; + await wrapped.openResource!('x-ggsql-result:ggsql-abc/0f9e'); + await tick(); + await tick(); + assert.deepStrictEqual(fake.calls, ['result_open']); + wrapped.dispose(); + }); + test('a link reopens a result whose Data Explorer was closed', async () => { const fake = fakeSession('console'); const explorer = new ResultExplorer(() => { }); diff --git a/src/reader/dialects/bigquery.rs b/src/reader/dialects/bigquery.rs index dc221c0df..d695a1e91 100644 --- a/src/reader/dialects/bigquery.rs +++ b/src/reader/dialects/bigquery.rs @@ -16,6 +16,12 @@ use crate::reader::SqlDialect; pub struct BigQueryDialect; impl SqlDialect for BigQueryDialect { + // RE2, unanchored. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let pattern = super::inline_regex_flag(self, pattern, case_sensitive); + Some(format!("REGEXP_CONTAINS({expr}, {pattern})")) + } + fn sql_string_literal(&self, value: &str) -> String { super::backslash_quote_literal(value) } diff --git a/src/reader/dialects/clickhouse.rs b/src/reader/dialects/clickhouse.rs index 463ec78e9..930188ada 100644 --- a/src/reader/dialects/clickhouse.rs +++ b/src/reader/dialects/clickhouse.rs @@ -30,6 +30,12 @@ fn float_args(exprs: &[&str]) -> String { } impl SqlDialect for ClickHouseDialect { + // RE2, unanchored. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let pattern = super::inline_regex_flag(self, pattern, case_sensitive); + Some(format!("match({expr}, {pattern})")) + } + fn sql_string_literal(&self, value: &str) -> String { super::backslash_quote_literal(value) } diff --git a/src/reader/dialects/databricks.rs b/src/reader/dialects/databricks.rs index 478e61506..172f3bb5a 100644 --- a/src/reader/dialects/databricks.rs +++ b/src/reader/dialects/databricks.rs @@ -11,6 +11,12 @@ use crate::reader::SqlDialect; pub struct DatabricksDialect; impl SqlDialect for DatabricksDialect { + // Java regular expressions, unanchored. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let pattern = super::inline_regex_flag(self, pattern, case_sensitive); + Some(format!("{expr} RLIKE {pattern}")) + } + fn sql_string_literal(&self, value: &str) -> String { super::backslash_quote_literal(value) } diff --git a/src/reader/dialects/datafusion.rs b/src/reader/dialects/datafusion.rs index cd843cff8..24ff0f7f9 100644 --- a/src/reader/dialects/datafusion.rs +++ b/src/reader/dialects/datafusion.rs @@ -13,6 +13,12 @@ use crate::reader::SqlDialect; pub struct DataFusionDialect; impl SqlDialect for DataFusionDialect { + // Postgres-style operators. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let op = if case_sensitive { "~" } else { "~*" }; + Some(format!("{expr} {op} {pattern}")) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("DOUBLE"), diff --git a/src/reader/dialects/druid.rs b/src/reader/dialects/druid.rs index ea1b77a5a..e6203ac10 100644 --- a/src/reader/dialects/druid.rs +++ b/src/reader/dialects/druid.rs @@ -14,6 +14,12 @@ use crate::reader::SqlDialect; pub struct DruidDialect; impl SqlDialect for DruidDialect { + // Java regular expressions, unanchored. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let pattern = super::inline_regex_flag(self, pattern, case_sensitive); + Some(format!("REGEXP_LIKE({expr}, {pattern})")) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("DOUBLE"), diff --git a/src/reader/dialects/mod.rs b/src/reader/dialects/mod.rs index 11ae4a236..dc6c91ed3 100644 --- a/src/reader/dialects/mod.rs +++ b/src/reader/dialects/mod.rs @@ -105,6 +105,18 @@ pub(crate) fn nulls_order_by_case(expr: &str, ascending: bool, nulls_last: bool) format!("CASE WHEN {expr} IS NULL THEN {null} ELSE {other} END, {expr} {direction}") } +/// The SQL string literal `pattern` with an inline `(?i)` or `(?-i)` flag in +/// front, for backends whose regular expressions take case sensitivity only +/// that way (RE2, Java, ICU, PCRE). +pub(crate) fn inline_regex_flag( + dialect: &D, + pattern: &str, + case_sensitive: bool, +) -> String { + let flag = if case_sensitive { "(?-i)" } else { "(?i)" }; + format!("CONCAT({}, {pattern})", dialect.sql_string_literal(flag)) +} + /// `1`/`0` boolean literal for backends without a real BOOLEAN literal /// (SQL Server, Oracle, SQLite). pub(crate) fn one_zero_boolean_literal(value: bool) -> String { @@ -247,6 +259,41 @@ mod tests { assert_eq!(df.height(), 1); } + #[test] + fn regex_search_is_unanchored_and_takes_case_where_the_backend_has_it() { + let regex = |scheme: &str, cs: bool| { + dialect_for_scheme(scheme) + .unwrap() + .sql_regex_match("c", "'^a'", cs) + }; + assert_eq!( + regex("bigquery", false).unwrap(), + r"REGEXP_CONTAINS(c, CONCAT('(?i)', '^a'))" + ); + assert_eq!( + regex("mysql", true).unwrap(), + r"c REGEXP CONCAT('(?-i)', '^a')" + ); + assert_eq!( + regex("snowflake", false).unwrap(), + r"REGEXP_INSTR(c, '^a', 1, 1, 0, 'i') > 0" + ); + assert_eq!(regex("oracle", true).unwrap(), r"REGEXP_LIKE(c, '^a', 'c')"); + for scheme in [ + "clickhouse", + "trino", + "databricks", + "redshift", + "postgres", + "duckdb", + ] { + assert!(regex(scheme, false).is_some(), "{scheme}"); + } + for scheme in ["sqlite", "mssql"] { + assert!(regex(scheme, false).is_none(), "{scheme}"); + } + } + /// Evaluate the CASE emulations against DuckDB's native /// `GREATEST`/`LEAST` so the left-folded chain is checked as executed /// SQL, not just as a string. @@ -414,7 +461,7 @@ mod tests { .as_deref(), Some("c ~ '^a'") ); - assert!(dialect_for_scheme("mysql") + assert!(dialect_for_scheme("sqlite") .unwrap() .sql_regex_match("c", "'^a'", true) .is_none()); diff --git a/src/reader/dialects/mysql.rs b/src/reader/dialects/mysql.rs index 12129044d..fdc4c7488 100644 --- a/src/reader/dialects/mysql.rs +++ b/src/reader/dialects/mysql.rs @@ -10,6 +10,12 @@ use crate::reader::SqlDialect; pub struct MySqlDialect; impl SqlDialect for MySqlDialect { + // `REGEXP` is unanchored on MySQL 8 (ICU) and MariaDB (PCRE), and both read `(?i)` and `(?-i)`; its own case sensitivity follows the collation. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let pattern = super::inline_regex_flag(self, pattern, case_sensitive); + Some(format!("{expr} REGEXP {pattern}")) + } + // No `NULLS FIRST` / `NULLS LAST`. fn sql_order_term(&self, expr: &str, ascending: bool, nulls_last: bool) -> String { super::nulls_order_by_case(expr, ascending, nulls_last) diff --git a/src/reader/dialects/oracle.rs b/src/reader/dialects/oracle.rs index b2e8aef52..cffc68e95 100644 --- a/src/reader/dialects/oracle.rs +++ b/src/reader/dialects/oracle.rs @@ -15,6 +15,12 @@ use crate::reader::SqlDialect; pub struct OracleDialect; impl SqlDialect for OracleDialect { + // Unanchored. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let flags = if case_sensitive { "c" } else { "i" }; + Some(format!("REGEXP_LIKE({expr}, {pattern}, '{flags}')")) + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("BINARY_DOUBLE"), diff --git a/src/reader/dialects/redshift.rs b/src/reader/dialects/redshift.rs index 4c0e3a9bc..c24bbb59d 100644 --- a/src/reader/dialects/redshift.rs +++ b/src/reader/dialects/redshift.rs @@ -17,6 +17,12 @@ use crate::reader::SqlDialect; pub struct RedshiftDialect; impl SqlDialect for RedshiftDialect { + // Unanchored, with the case flag as a parameter. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let flags = if case_sensitive { "c" } else { "i" }; + Some(format!("REGEXP_COUNT({expr}, {pattern}, 1, '{flags}') > 0")) + } + fn sql_string_literal(&self, value: &str) -> String { super::backslash_quote_literal(value) } diff --git a/src/reader/dialects/snowflake.rs b/src/reader/dialects/snowflake.rs index c3235d1e2..c58712d6d 100644 --- a/src/reader/dialects/snowflake.rs +++ b/src/reader/dialects/snowflake.rs @@ -11,6 +11,14 @@ use crate::reader::SqlDialect; pub struct SnowflakeDialect; impl SqlDialect for SnowflakeDialect { + // `REGEXP_LIKE` matches the whole string here, so search with `REGEXP_INSTR`. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let flags = if case_sensitive { "c" } else { "i" }; + Some(format!( + "REGEXP_INSTR({expr}, {pattern}, 1, 1, 0, '{flags}') > 0" + )) + } + fn sql_string_literal(&self, value: &str) -> String { super::backslash_quote_literal(value) } diff --git a/src/reader/dialects/trino.rs b/src/reader/dialects/trino.rs index c616e90a4..127372107 100644 --- a/src/reader/dialects/trino.rs +++ b/src/reader/dialects/trino.rs @@ -15,6 +15,12 @@ use crate::reader::SqlDialect; pub struct TrinoDialect; impl SqlDialect for TrinoDialect { + // Unanchored. + fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { + let pattern = super::inline_regex_flag(self, pattern, case_sensitive); + Some(format!("regexp_like({expr}, {pattern})")) + } + fn supports_spatial(&self) -> bool { true } From 0af32b69dfb48b072e7f8f3679be2112355ae66d Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Fri, 20 Nov 2026 12:44:11 -0600 Subject: [PATCH 6/8] Keep a result whose open fails after making room Making room can drop older unopened results from the front of the queue, so putting the result back at its old index could pass the end and panic. It goes back at its index or, past the end, last. Co-Authored-By: Claude Opus 5.5 --- ggsql-jupyter/src/results/mod.rs | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs index f42070a64..8f837667a 100644 --- a/ggsql-jupyter/src/results/mod.rs +++ b/ggsql-jupyter/src/results/mod.rs @@ -224,18 +224,20 @@ impl ResultStore { pub fn open(&mut self, key: &str) -> Result { if let Some(i) = self.pending.iter().position(|p| p.key == key) { let pending = self.pending.remove(i).expect("position is in range"); + // Making room may drop older results from the front, so a result + // put back goes where it was or, past the end, last. // Out of the queue, so making room cannot drop it; its copy is // budgeted afresh, with the row column the copy adds. let need = pending.bytes + pending.df.height() as u64 * ROW_BYTES; let fits = match self.make_room(need) { Ok(fits) => fits, Err(e) => { - self.pending.insert(i, pending); + self.pending.insert(i.min(self.pending.len()), pending); return Err(e); } }; if !fits { - self.pending.insert(i, pending); + self.pending.insert(i.min(self.pending.len()), pending); bail!( "Opening this result needs more memory than the data explorer's budget \ has left ({} bytes). Close another Data Explorer tab and try again.", @@ -249,7 +251,7 @@ impl ResultStore { Err(e) => { // Kept, so the error says what went wrong each time // rather than that the result is gone. - self.pending.insert(i, pending); + self.pending.insert(i.min(self.pending.len()), pending); return Err(e); } } @@ -687,6 +689,27 @@ mod tests { } } + #[test] + fn test_a_failed_open_after_making_room_keeps_the_result() { + let n = 200_000; + // Room for both results as kept, and for the second's copy, which + // adds a row column, only once the first has gone. + let mut store = ResultStore::new(2 * bytes_of(&numbers(n)) + 1).unwrap(); + let first = keep(&mut store, numbers(n), "a".into(), String::new()); + let second = keep(&mut store, numbers(n), "b".into(), String::new()); + // Too little memory for DuckDB to take the copy. + store + .cache + .backend() + .execute_sql("SET memory_limit = '256KB'") + .unwrap(); + let err = store.open(&second).unwrap_err().to_string(); + assert!(!err.contains("no longer available"), "{err}"); + // The first went to make room; the second is still kept. + assert!(store.pending.iter().all(|p| p.key != first)); + assert!(store.pending.iter().any(|p| p.key == second)); + } + #[test] fn test_sql_leaves_out_a_filter_the_cache_rejected() { let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); From aa3c5ad365418bba85d8f7439fcd5657a86e0650 Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Fri, 20 Nov 2026 13:31:31 -0600 Subject: [PATCH 7/8] Address Thomas-lens review of the results API - Validate export columns before anything else, so a request with no rows and a bad column is an error rather than a kernel panic. - Cut a console table short only when its result was kept, since only then does a link to the rest follow it. - Keep Positron out of the kernel's results wording; budget errors now say how much open views hold. - Case-sensitive LIKE for SQLite (GLOB), MySQL (binary) and SQL Server (case-sensitive collation); `sql_is_empty_string`, NULL on Oracle. - CacheTables: `put`/`put_pinned`, `contains`, `evict_to` returns the evicted keys, only an existing entry can be pinned; `MemoEntry` and `lookup` are crate-private. - Format timezone-aware timestamps in `array_util::value_to_display`. - Share `kind_of` with the Connections-pane explorer, fixing its substring type matching. - Internal names through `ggsql::naming`; numbers read as numbers in profiles; bounded profile requests. - `ggsql.resultsMemory` setting; overflow-safe `parse_human_bytes`. - Drop the results benchmark example from this PR. - Warm the renderer in a timing-sensitive plot test. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +- doc/get_started/tooling/positron-vscode.qmd | 2 +- ggsql-jupyter/AGENTS.md | 6 +- ggsql-jupyter/examples/results_bench.rs | 289 -------------------- ggsql-jupyter/src/data_explorer.rs | 55 ++-- ggsql-jupyter/src/display.rs | 88 +++--- ggsql-jupyter/src/kernel.rs | 6 +- ggsql-jupyter/src/plot/backend.rs | 6 + ggsql-jupyter/src/results/export.rs | 2 +- ggsql-jupyter/src/results/filter.rs | 20 +- ggsql-jupyter/src/results/mod.rs | 169 ++++++++---- ggsql-jupyter/src/results/view.rs | 171 ++++++++---- ggsql-jupyter/tests/test_integration.py | 2 +- ggsql-vscode/AGENTS.md | 5 +- ggsql-vscode/package.json | 8 +- ggsql-vscode/src/manager.ts | 14 + ggsql-vscode/src/test/manager.test.ts | 14 +- ggsql-vscode/src/test/resultLinks.test.ts | 3 +- src/AGENTS.md | 2 +- src/array_util.rs | 33 ++- src/naming.rs | 33 +++ src/reader/cache.rs | 108 +++++--- src/reader/dialects/mod.rs | 111 ++++++-- src/reader/dialects/mssql.rs | 9 +- src/reader/dialects/mysql.rs | 14 + src/reader/dialects/oracle.rs | 9 + src/reader/dialects/sqlite.rs | 12 + src/reader/mod.rs | 9 + 28 files changed, 636 insertions(+), 577 deletions(-) delete mode 100644 ggsql-jupyter/examples/results_bench.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fda9154f..4c613e71c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,8 @@ determines whether text boxes get drawn and additional settings `padding`, through a frontend-neutral results API, reached with `call_method` on the UI comm and turned on by the frontend (`--keep-results`), which keeps results in an in-memory cache of its own within one byte budget - (`--results-max-bytes`, default 1 GB) and never changes the session's - reader or dialect (#595). + (`--results-max-bytes`, default 1 GB, or the `ggsql.resultsMemory` + setting) and never changes the session's reader or dialect (#595). ### Changed @@ -48,10 +48,11 @@ determines whether text boxes get drawn and additional settings `padding`, - The CLI now always prints query warnings (e.g. extra `VISUALISE`/`TABULATE` statements being ignored) to stderr, not only with `-v`. - Replaced `CLAUDE.md` files with `AGENTS.md` files. -- A Positron console now shows a long query result's first 10 rows, followed - by a count of the rest, rather than up to 100; a result of up to 20 rows is - still shown whole. The number is set by the `ggsql.consoleRows` setting - (#595). +- A Positron console now shows a long query result that can be opened in the + Data Explorer as its first 10 rows, followed by a count of the rest, rather + than up to 100; a result of up to 20 rows is still shown whole, and one that + could not be kept still shows up to 100. The number is set by the + `ggsql.consoleRows` setting (#595). ### Fixed diff --git a/doc/get_started/tooling/positron-vscode.qmd b/doc/get_started/tooling/positron-vscode.qmd index 4662f1a41..862728b14 100644 --- a/doc/get_started/tooling/positron-vscode.qmd +++ b/doc/get_started/tooling/positron-vscode.qmd @@ -34,7 +34,7 @@ Executing ggsql statements containing [`VISUALISE`](../../syntax/clause/visualis A query without `VISUALISE` shows its result as a table in the console, followed by an "Open in Data Explorer" link. A result of more than 20 rows shows only its first 10, with a count of the rest. To show more or fewer, change the `ggsql.consoleRows` setting: a result of up to twice that many rows is shown in full. The change applies to consoles started afterwards. Clicking the link opens the result in Positron's Data Explorer, where you can sort, filter, and profile it, and use "Convert to Code" to turn your sorts and filters back into SQL. -The Data Explorer works on a snapshot of the result, so it doesn't change when the underlying tables do. Run the query again to see new data. The console keeps recent results available this way, up to 1 GB in all. Results open in a Data Explorer are always kept, and older ones make way for new ones, so the link under an old result may report that it is no longer available. If the results you have open use the whole 1 GB, a new result is shown in the console without a link, and sorting or filtering a large one may ask you to close another Data Explorer tab first. +The Data Explorer works on a snapshot of the result, so it doesn't change when the underlying tables do. Run the query again to see new data. The console keeps recent results available this way, up to 1 GB in all by default; the `ggsql.resultsMemory` setting changes that, for consoles started afterwards. Results open in a Data Explorer are always kept, and older ones make way for new ones, so the link under an old result may report that it is no longer available. If the results you have open use all of that memory, a new result is shown in the console without a link, and up to 100 of its rows, and sorting or filtering a large one may ask you to close another Data Explorer tab first. ## Editor and cells diff --git a/ggsql-jupyter/AGENTS.md b/ggsql-jupyter/AGENTS.md index 8cbb36207..778ca2acc 100644 --- a/ggsql-jupyter/AGENTS.md +++ b/ggsql-jupyter/AGENTS.md @@ -9,7 +9,6 @@ End-user installation and usage live in [`README.md`](README.md). End-user noteb ``` ggsql-jupyter/ ├── Cargo.toml Rust binary + library, depends on ggsql with duckdb + svg/pdf + raster-plots -├── examples/results_bench.rs Results API benchmark (DuckDB cache vs. Arrow row indices) ├── pyproject.toml maturin config (bindings = "bin") for the PyPI wheel ├── README.md User-facing install + usage ├── src/ @@ -187,19 +186,22 @@ So `RenderHints::canvas` picks a height (golden ratio, close to ggplot2's defaul `results/` keeps recent query results and answers a small, frontend-neutral set of calls on them (`methods.rs` documents each: `result_open`, `result_set_filters`, `result_set_sort`, `result_page`, `result_profile`, `result_export`, `result_sql`, `result_close`). Nothing in it knows about Positron: the ggsql extension translates Positron's Data Explorer protocol into these calls (see [`ggsql-vscode/AGENTS.md`](../ggsql-vscode/AGENTS.md)). The kernel's only Positron-specific part is the route: a `call_method` request on the `positron.ui` comm — what an extension's `session.callMethod(name, params)` sends — whose method starts with `result_` goes to `methods::call`, and its answer or error comes back as the JSON-RPC reply (errors with `plot::comm::RpcError`'s encoding). A `call_method` for any other name is an error, like an unknown plot-comm method, so a caller expecting something the kernel lacks fails visibly; any other UI request gets `result: null`. +- **A console cuts a long table short only when it was kept** (`RenderHints::kept`): only then does a link to the rest follow it. A result that could not be kept — the budget full, or a frontend that did not ask — shows up to 100 rows, as anywhere else. - **Which results are kept.** Only when the frontend asks, with `--keep-results` or `GGSQL_KEEP_RESULTS` — the extension sets it for a console session whose results it can open — so a session nobody explores holds nothing beyond what it displays. Then every tabular result of a non-silent execution is kept, and its key — unguessable hex, so a stale link from another kernel process names nothing — goes in the `execute_result`'s `metadata` as `{"ggsql": {"result": key}}`; the kernel prints no link itself. Unopened results are held as their Arrow data. The store and its DuckDB are created on first use, so a session that keeps nothing opens no database for them. - **One byte budget covers everything kept**, `--results-max-bytes` / `GGSQL_RESULTS_MAX_BYTES` (default 1 GB, by Arrow's estimate of the data): unopened results, the cache's tables, pinned ones included, and every view's order table. Making room drops closed results first, then the oldest unopened ones — and only once it is known the new bytes fit beside what open views hold, so a result too big to keep drops nothing. An open view is never pulled from under its frontend, so when open views alone fill the budget a new result is not kept (it is shown without a key), and opening a result or a sort or filter that needs room fails with an error saying so. Opening reserves the copy's row column as well as the result, and any rebuild of an order table reserves a new one, including clearing the sort of a filtered view or the filters of a sorted one. - **The cache's DuckDB has a hard memory limit of twice the budget** (`memory_limit_for`, at least 64 MB). The budget counts what is kept, by Arrow's estimate; DuckDB holds a table less compactly than that and needs working memory on top to sort and profile, so the limit is what bounds the process. A budget too large to set as a limit leaves DuckDB's default, most of the machine's memory. - **Opening copies the result into the results cache.** A `CacheTables` (the caching reader's bookkeeping, `ggsql::reader::cache`) on its own in-memory DuckDB, which `ResultStore` creates itself. The session's reader is never wrapped, so the user's dialect is untouched. The copy adds `__ggsql_row__`, each row's position in the query's output, and drops the Arrow data, so a result is held once. It stays **pinned** while any view of it is open; once closed, it goes when the budget needs the room, and reopening before then reuses it. A result can have several views, each with its own filters and sort. - **Views end with the UI comm.** Closing it closes every view, since nothing could close them once it has gone, and so does opening a new one: a frontend that reloaded or crashed opens a new UI comm without closing the old. That is what releases its pins. +- **The cache is always DuckDB.** `ResultStore` opens it itself, so `results/` writes DuckDB SQL for it (`DESCRIBE`, `SET`, ANSI quoting through `naming::quote_ident`, internal names through `ggsql::naming`). Type names, booleans and quantiles come from the cache reader's own dialect, which is DuckDB's; the filter and sort SQL in `filter.rs` takes a dialect because `result_sql` uses the same code to write SQL for the session's database. - **Nothing is written to disk.** The cache sets `temp_directory = ''`, so DuckDB cannot spill: a query that needs more memory than the limit fails with DuckDB's out-of-memory error, and the view it was for is left as it was. - **Sorting and filtering build an order table** of `(pos, __ggsql_row__)` per view (`view.rs`); a page joins it back to the result by `pos`, so scrolling never sorts again. Ties break on `__ggsql_row__`, so paging is stable and clearing the sort restores the query's own order. An unfiltered, unsorted view has none and pages by `__ggsql_row__` directly. A new sort or set of filters is built under a new table name and only then replaces the old one, so one that fails leaves the view as it was. +- **Profiles are bounded too**: at most `MAX_FREQUENCY_VALUES` values in a frequency table, `MAX_QUANTILES` quantiles (each its own query) and `MAX_HISTOGRAM_BINS` bins. - **A page is bounded, and so is an export.** A page holds at most `MAX_PAGE_ROWS`, and an export at most `MAX_EXPORT_ROWS` (100,000), since it is built as text on the message loop, outside DuckDB's memory limit; copying a whole column of a longer result is an error that says so. For both, a range is cut short at the end of the view, and an index outside it is an error, so a cell never lands in another row's place. Scattered indices fetch just those rows. - **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, a date, datetime or time as ISO 8601 without an offset (it becomes the dialect's typed literal, `sql_date_literal` and friends, since not every backend compares a string with a temporal column), anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, string and boolean literals, `LIKE` and regular expressions all come from the dialect the SQL is for (`SqlDialect::sql_string_literal`, `sql_like`, `sql_regex_match` — `None` where a backend has no regular expressions, as SQLite and SQL Server), so the same filter is written correctly for a backend that reads backslashes in literals, has no `ESCAPE` clause, or has more `LIKE` wildcards. Sort keys put NULLs last in either direction through `sql_order_term`, as the cache's DuckDB does, so exported SQL sorts as the view did. - **Kinds are DuckDB type names matched exactly** (`kind_of`): a column type that only contains `INT`, like `POINT`, is not an integer, and an `ENUM` is a string. - **`result_sql` writes SQL for the session's database**, not the cache: the cell's statements before the last as they were (the whole cell is one statement when it does not parse cleanly), then the last as a derived table with the view's filters and sort, in the session's dialect. A last statement that is not a query (`DESCRIBE`, `SHOW`, …) cannot be a derived table, so it is written as it was, with a comment saying the filters and sort were left out. It is in the kernel because only the kernel knows that dialect. A filter it cannot express there, or one on a column whose name the query repeats (renamed `a_1` in the cache, but ambiguous in SQL), is left out with a `--` comment. -`examples/results_bench.rs` measures it. At 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 3.4 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. +Measured at 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 3.4 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. ## Build & install diff --git a/ggsql-jupyter/examples/results_bench.rs b/ggsql-jupyter/examples/results_bench.rs deleted file mode 100644 index ee236bda2..000000000 --- a/ggsql-jupyter/examples/results_bench.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! Benchmark for the results API: the DuckDB results cache it uses, against -//! sorting and filtering the Arrow data directly with row indices. -//! -//! ```sh -//! cargo run --release -p ggsql-jupyter --example results_bench -- store 10000000 -//! cargo run --release -p ggsql-jupyter --example results_bench -- arrow 10000000 -//! ``` -//! -//! Each mode runs in its own process so the resident-memory figures don't mix. -//! `store` also opens and closes a result repeatedly, to check that memory -//! levels off rather than growing with every cycle. - -use arrow::array::{ - Array, ArrayRef, BooleanArray, Date32Array, Float64Array, Int64Array, StringArray, UInt32Array, -}; -use arrow::compute::kernels::cmp::{gt_eq, lt_eq}; -use arrow::compute::kernels::comparison::contains; -use arrow::compute::{and, filter, lexsort_to_indices, take, SortColumn, SortOptions}; -use arrow::datatypes::{DataType, Field, Schema}; -use arrow::record_batch::RecordBatch; -use ggsql::reader::connection::reader_from_uri; -use ggsql::DataFrame; -use ggsql_jupyter::results::methods::{self, Session}; -use ggsql_jupyter::results::ResultStore; -use serde_json::{json, Value}; -use std::sync::Arc; -use std::time::Instant; - -const CATEGORIES: &[&str] = &[ - "alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", - "kilo", "lima", "mike", "november", "oscar", "papa", "quebec", "romeo", "sierra", "tango", -]; - -struct Lcg(u64); -impl Lcg { - fn next(&mut self) -> u64 { - self.0 = self - .0 - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - self.0 >> 33 - } -} - -fn make_data(n: usize) -> RecordBatch { - let mut rng = Lcg(42); - let id: Int64Array = (0..n as i64).collect(); - let x: Float64Array = (0..n) - .map(|_| { - let v = rng.next(); - (v % 50 != 0).then(|| (v % 1_000_000) as f64 / 7.0) - }) - .collect(); - let category: StringArray = (0..n) - .map(|_| Some(CATEGORIES[(rng.next() % CATEGORIES.len() as u64) as usize])) - .collect(); - let name: StringArray = (0..n) - .map(|_| { - let len = 12 + (rng.next() % 20) as usize; - let s: String = (0..len) - .map(|_| (b'a' + (rng.next() % 26) as u8) as char) - .collect(); - Some(s) - }) - .collect(); - let day: Date32Array = (0..n).map(|_| Some((rng.next() % 20_000) as i32)).collect(); - let schema = Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("x", DataType::Float64, true), - Field::new("category", DataType::Utf8, false), - Field::new("name", DataType::Utf8, false), - Field::new("day", DataType::Date32, false), - ]); - RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(id), - Arc::new(x), - Arc::new(category), - Arc::new(name), - Arc::new(day), - ], - ) - .unwrap() -} - -fn rss_mb() -> f64 { - let out = std::process::Command::new("ps") - .args(["-o", "rss=", "-p", &std::process::id().to_string()]) - .output() - .unwrap(); - String::from_utf8_lossy(&out.stdout) - .trim() - .parse::() - .unwrap_or(0.0) - / 1024.0 -} - -fn time(label: &str, f: impl FnOnce() -> T) -> T { - let t = Instant::now(); - let r = f(); - println!( - " {label:<40} {:>9.1} ms rss {:>7.0} MB", - t.elapsed().as_secs_f64() * 1000.0, - rss_mb() - ); - r -} - -/// Sorting and filtering the Arrow data directly, for comparison. -fn run_arrow(batch: RecordBatch) { - let n = batch.num_rows(); - let rows: ArrayRef = Arc::new(UInt32Array::from_iter_values(0..n as u32)); - let col = |name: &str| batch.column(batch.schema().index_of(name).unwrap()).clone(); - let asc = Some(SortOptions { - descending: false, - nulls_first: false, - }); - // Stable sorts: the row index is the last key. - let sort = |keys: &[&str]| { - let mut columns: Vec = keys - .iter() - .map(|k| SortColumn { - values: col(k), - options: asc, - }) - .collect(); - columns.push(SortColumn { - values: rows.clone(), - options: asc, - }); - lexsort_to_indices(&columns, None).unwrap() - }; - - let sort1 = time("sort x", || sort(&["x"])); - time("sort category, x", || sort(&["category", "x"])); - time("sort x (sort_to_indices, unstable)", || { - arrow::compute::sort_to_indices(&col("x"), asc, None).unwrap() - }); - time("filter name contains 'abc'", || { - let name = col("name"); - let name = name.as_any().downcast_ref::().unwrap(); - let mask = contains(name, &StringArray::new_scalar("abc")).unwrap(); - filter(&rows, &mask).unwrap() - }); - time("filter 1000 <= x <= 50000", || { - let x = col("x"); - let mask: BooleanArray = and( - >_eq(&x, &Float64Array::new_scalar(1000.0)).unwrap(), - <_eq(&x, &Float64Array::new_scalar(50000.0)).unwrap(), - ) - .unwrap(); - filter(&rows, &mask).unwrap() - }); - time("page 100 rows mid sorted view", || { - let page = sort1.slice(n / 2, 100); - batch - .columns() - .iter() - .map(|c| take(c, &page, None).unwrap()) - .collect::>() - }); -} - -/// The results API itself, through its JSON calls. -fn run_store(batch: RecordBatch, cycles: usize) { - let n = batch.num_rows(); - let session_reader = reader_from_uri("duckdb://memory").unwrap(); - let session = Session { - reader: &*session_reader, - }; - // Unbounded, so the benchmark measures the work rather than the budget. - let mut store = ResultStore::new(u64::MAX).unwrap(); - let key = store - .insert( - DataFrame::from_record_batch(batch), - "bench".into(), - "SELECT 1".into(), - ) - .unwrap() - .expect("an unbounded store keeps every result"); - let mut call = |method: &str, params: Value| { - methods::call(&mut store, &session, method, ¶ms) - .unwrap_or_else(|e| panic!("{method}: {e}")) - }; - - let view = time("open (copy into the cache)", || { - call("result_open", json!({ "key": key })) - })["view"] - .as_str() - .unwrap() - .to_string(); - let sort = |keys: Value| json!({ "view": view, "keys": keys }); - let filters = |f: Value| json!({ "view": view, "filters": f }); - let mid = json!({ "view": view, "rows": {"first": n / 2, "last": n / 2 + 99}, "columns": [0, 1, 2, 3, 4] }); - - time("page 100 rows mid unsorted", || { - call("result_page", mid.clone()) - }); - time("sort x", || { - call("result_set_sort", sort(json!([{"column": 1}]))) - }); - time("page 100 rows mid sorted view", || { - call("result_page", mid.clone()) - }); - time("sort category, x", || { - call( - "result_set_sort", - sort(json!([{"column": 2}, {"column": 1}])), - ) - }); - call("result_set_sort", sort(json!([]))); - time("filter name contains 'abc'", || { - call( - "result_set_filters", - filters( - json!([{"type": "text", "column": 3, "match": "contains", "term": "abc", "case_sensitive": true}]), - ), - ) - }); - time("filter 1000 <= x <= 50000", || { - call( - "result_set_filters", - filters(json!([{"type": "between", "column": 1, "low": "1000", "high": "50000"}])), - ) - }); - time("filter range then sort x", || { - call("result_set_sort", sort(json!([{"column": 1}]))) - }); - time("profile x (summary + histogram)", || { - call( - "result_profile", - json!({"view": view, "column": 1, "profiles": {"null_count": true, "summary": true, "histogram": {"bins": 20, "quantiles": []}}}), - ) - }); - call("result_close", json!({ "view": view })); - println!( - " after close rss {:>7.0} MB", - rss_mb() - ); - - // Reopen, sort, page and close repeatedly: memory must level off. - for cycle in 1..=cycles { - let view = call("result_open", json!({ "key": key }))["view"] - .as_str() - .unwrap() - .to_string(); - call( - "result_set_sort", - json!({ "view": view, "keys": [{"column": 2}, {"column": 1}] }), - ); - call( - "result_set_filters", - json!({ "view": view, "filters": [{"type": "between", "column": 1, "low": "1000", "high": "50000"}] }), - ); - call( - "result_page", - json!({ "view": view, "rows": {"first": 0, "last": 99}, "columns": [0, 1] }), - ); - call("result_close", json!({ "view": view })); - println!( - " cycle {cycle:>2} open/sort/filter/page/close rss {:>7.0} MB", - rss_mb() - ); - } -} - -fn main() { - let args: Vec = std::env::args().collect(); - let mode = args.get(1).map(String::as_str).unwrap_or("store"); - let n: usize = args - .get(2) - .and_then(|s| s.parse().ok()) - .unwrap_or(1_000_000); - let cycles: usize = args.get(3).and_then(|s| s.parse().ok()).unwrap_or(10); - println!("mode={mode} rows={n} baseline rss {:.0} MB", rss_mb()); - let batch = time("generate data", || make_data(n)); - let bytes: usize = batch - .columns() - .iter() - .map(|c| c.get_array_memory_size()) - .sum(); - println!(" arrow data size {:.0} MB", bytes as f64 / 1e6); - match mode { - "arrow" => run_arrow(batch), - "store" => run_store(batch, cycles), - _ => panic!("mode must be store or arrow"), - } -} diff --git a/ggsql-jupyter/src/data_explorer.rs b/ggsql-jupyter/src/data_explorer.rs index 9ddc137a8..b21981de7 100644 --- a/ggsql-jupyter/src/data_explorer.rs +++ b/ggsql-jupyter/src/data_explorer.rs @@ -917,46 +917,19 @@ fn sql_type_to_display(type_name: &str) -> &'static str { } } - // Simple type names (DuckDB, PostgreSQL, SQLite, etc.) - let upper = type_name.to_uppercase(); - let upper = upper.as_str(); - - if upper.contains("INT") { - return "integer"; - } - if upper.contains("FLOAT") - || upper.contains("DOUBLE") - || upper.contains("REAL") - || upper.contains("NUMERIC") - || upper.contains("DECIMAL") - { - return "floating"; - } - if upper.contains("BOOL") { - return "boolean"; - } - if upper.contains("TIMESTAMP") || upper.contains("DATETIME") { - return "datetime"; - } - if upper.contains("DATE") { - return "date"; - } - if upper.contains("TIME") { - return "time"; - } - if upper.contains("CHAR") - || upper.contains("TEXT") - || upper.contains("STRING") - || upper.contains("VARCHAR") - || upper.contains("CLOB") - { - return "string"; - } - if upper.contains("BLOB") || upper.contains("BINARY") || upper.contains("BYTE") { - return "string"; + // Simple type names, matched exactly as the results API matches them. + match crate::results::kind_of(type_name) { + "unknown" => { + // Binary data is shown as text here. + let upper = type_name.trim().to_uppercase(); + let base = upper.split('(').next().unwrap_or("").trim(); + match base { + "BLOB" | "BYTEA" | "BINARY" | "VARBINARY" | "BYTES" | "LONGBLOB" => "string", + _ => "unknown", + } + } + kind => kind, } - - "unknown" } /// Clean up a raw type name for display in the schema response. @@ -1014,5 +987,9 @@ mod tests { assert_eq!(sql_type_to_display("TIME"), "time"); assert_eq!(sql_type_to_display("BLOB"), "string"); assert_eq!(sql_type_to_display("UNKNOWN_TYPE"), "unknown"); + // Matched exactly, not by substring. + assert_eq!(sql_type_to_display("POINT"), "unknown"); + assert_eq!(sql_type_to_display("INT UNSIGNED"), "integer"); + assert_eq!(sql_type_to_display("character varying(20)"), "string"); } } diff --git a/ggsql-jupyter/src/display.rs b/ggsql-jupyter/src/display.rs index 1877f4e3b..640450335 100644 --- a/ggsql-jupyter/src/display.rs +++ b/ggsql-jupyter/src/display.rs @@ -85,6 +85,9 @@ pub struct RenderHints { pub pixel_ratio: Option, /// How many rows of a long result a console shows (`--console-rows`). pub console_rows: usize, + /// Whether the result was kept for the results API, so the frontend can + /// follow it with a link to the whole of it. + pub kept: bool, } impl Default for RenderHints { @@ -94,6 +97,7 @@ impl Default for RenderHints { output_width_px: None, pixel_ratio: None, console_rows: DEFAULT_CONSOLE_ROWS, + kept: false, } } } @@ -295,48 +299,23 @@ fn format_dataframe(df: DataFrame, hints: &RenderHints) -> Value { /// How many of a result's `height` rows its HTML table shows. /// -/// A console result is followed by a link to the whole of it in the data -/// explorer, so it shows what R's tibble does: every row up to twice -/// `console_rows`, else the first `console_rows`, which keeps the link in -/// view. Anywhere else the table is all there is, so it shows up to 100. +/// A kept console result is followed by a link to the whole of it, so it +/// shows what R's tibble does: every row up to twice `console_rows`, else the +/// first `console_rows`, which keeps the link in view. Anywhere else, and for +/// a result that was not kept, the table is all there is, so it shows up to +/// 100. fn html_rows_shown(height: usize, hints: &RenderHints) -> usize { let n = hints.console_rows; - match hints.kind { - SessionKind::PositronConsole if height > n.saturating_mul(2) => n, - SessionKind::PositronConsole => height, - _ => height.min(100), + let linked = hints.kind == SessionKind::PositronConsole && hints.kept; + if !linked { + height.min(100) + } else if height > n.saturating_mul(2) { + n + } else { + height } } -/// A cell as a person reads it: `value_to_string`, except that dates are -/// written as dates rather than as the day or millisecond counts Arrow stores, -/// and timestamps with a time zone as timestamps in UTC with their offset. -pub(crate) fn cell_text(col: &arrow::array::ArrayRef, row: usize) -> String { - use arrow::array::Array; - use arrow::datatypes::DataType; - use arrow::util::display::{ArrayFormatter, FormatOptions}; - let formatted = |cell: &dyn Array| { - ArrayFormatter::try_new(cell, &FormatOptions::default()) - .map(|f| f.value(0).to_string()) - .ok() - }; - let text = match col.data_type() { - _ if col.is_null(row) => None, - DataType::Date32 | DataType::Date64 => formatted(&col.slice(row, 1)), - // Arrow here reads only fixed offsets as zones, not names such as - // the `UTC` DuckDB tags these with. The stored value is an instant, - // so retagging it as `+00:00` writes the same moment. - DataType::Timestamp(unit, Some(_)) => arrow::compute::cast( - &col.slice(row, 1), - &DataType::Timestamp(*unit, Some("+00:00".into())), - ) - .ok() - .and_then(|cell| formatted(&cell)), - _ => None, - }; - text.unwrap_or_else(|| ggsql::array_util::value_to_string(col, row)) -} - /// Convert DataFrame to an HTML table of its first `row_limit` rows fn dataframe_to_html(df: &DataFrame, row_limit: usize) -> String { let mut html = String::from("\n"); @@ -350,7 +329,7 @@ fn dataframe_to_html(df: &DataFrame, row_limit: usize) -> String { for i in 0..row_limit { html.push_str(""); for col in df.get_columns() { - let value = cell_text(col, i); + let value = ggsql::array_util::value_to_string(col, i); html.push_str(&format!("", escape_html(&value))); } html.push_str("\n"); @@ -375,7 +354,11 @@ fn dataframe_to_text(df: &ggsql::DataFrame, row_limit: usize) -> String { s.push_str(&names.join("\t")); s.push('\n'); for i in 0..row_limit { - let row: Vec = df.get_columns().iter().map(|c| cell_text(c, i)).collect(); + let row: Vec = df + .get_columns() + .iter() + .map(|c| ggsql::array_util::value_to_string(c, i)) + .collect(); s.push_str(&row.join("\t")); s.push('\n'); } @@ -431,9 +414,12 @@ mod tests { "SELECT TIMESTAMPTZ '2024-01-02 03:04:05+00' AS t, NULL::TIMESTAMPTZ AS missing", ) .unwrap(); - let text = cell_text(df.column("t").unwrap(), 0); + let text = ggsql::array_util::value_to_string(df.column("t").unwrap(), 0); assert_eq!(text, "2024-01-02T03:04:05Z"); - assert_eq!(cell_text(df.column("missing").unwrap(), 0), "null"); + assert_eq!( + ggsql::array_util::value_to_string(df.column("missing").unwrap(), 0), + "null" + ); } #[test] @@ -586,11 +572,13 @@ mod tests { #[test] fn test_html_rows_shown() { - let console = positron_console(); - // The console shows a short result whole, and a long one's first 10. + let console = kept_console(); + // The console shows a short kept result whole, and a long one's first 10. assert_eq!(html_rows_shown(20, &console), 20); assert_eq!(html_rows_shown(21, &console), 10); assert_eq!(html_rows_shown(5000, &console), 10); + // Not kept, it has no link to the rest, so it shows up to 100. + assert_eq!(html_rows_shown(5000, &positron_console()), 100); // Anywhere else, up to 100. assert_eq!(html_rows_shown(50, &positron_notebook()), 50); assert_eq!(html_rows_shown(5000, &positron_notebook()), 100); @@ -601,7 +589,7 @@ mod tests { fn test_console_rows_is_configurable() { let console = RenderHints { console_rows: 3, - ..positron_console() + ..kept_console() }; assert_eq!(html_rows_shown(6, &console), 6); assert_eq!(html_rows_shown(7, &console), 3); @@ -619,7 +607,7 @@ mod tests { .unwrap() .execute_sql("SELECT i FROM range(5000) t(i)") .unwrap(); - let bundle = format_dataframe(df, &positron_console()); + let bundle = format_dataframe(df, &kept_console()); let html = bundle["data"]["text/html"].as_str().unwrap(); assert_eq!(html.matches("").count(), 1 + 10 + 1, "{html}"); assert!(html.contains("... 4990 more rows"), "{html}"); @@ -633,7 +621,7 @@ mod tests { .unwrap(); let console = RenderHints { console_rows: 3, - ..positron_console() + ..kept_console() }; let bundle = format_dataframe(df, &console); let text = bundle["data"]["text/plain"].as_str().unwrap(); @@ -654,6 +642,14 @@ mod tests { } } + /// A console whose result was kept, so a link follows it. + fn kept_console() -> RenderHints { + RenderHints { + kept: true, + ..positron_console() + } + } + fn positron_notebook() -> RenderHints { RenderHints { kind: SessionKind::PositronNotebook, diff --git a/ggsql-jupyter/src/kernel.rs b/ggsql-jupyter/src/kernel.rs index 60af719ec..c90dcc2b2 100644 --- a/ggsql-jupyter/src/kernel.rs +++ b/ggsql-jupyter/src/kernel.rs @@ -376,7 +376,7 @@ impl KernelServer { let content = &parent.content; let code = content["code"].as_str().unwrap_or(""); let silent = content["silent"].as_bool().unwrap_or(false); - let hints = RenderHints { + let mut hints = RenderHints { console_rows: self.console_rows, ..RenderHints::from_request(&parent.header, content, self.session_mode) }; @@ -444,6 +444,10 @@ impl KernelServer { } } + // Only a kept result gets a link to the rest of it, so only a kept + // result is cut short. + hints.kept = retained.is_some(); + // Formatting renders the plot, so it is fallible and its failures are // the user's. Both stages feed one `Result` and share the error path // below; propagating out of the loop would end the process and take diff --git a/ggsql-jupyter/src/plot/backend.rs b/ggsql-jupyter/src/plot/backend.rs index 20f64936d..3571e94b1 100644 --- a/ggsql-jupyter/src/plot/backend.rs +++ b/ggsql-jupyter/src/plot/backend.rs @@ -582,6 +582,12 @@ mod tests { #[test] fn a_stored_plot_keeps_one_namespace_across_renders() { let backend = PlotBackend::without_raster(); + // `render_stored` waits only the pre-render budget, and a process's + // first render loads its fonts, which under a loaded parallel test + // run can take longer. + backend + .render_once(a_spec(), an_svg_request()) + .expect("the SVG path needs no adapter"); backend.store("comm-1".to_string(), a_spec()); let first = backend.render_stored("comm-1", an_svg_request()).unwrap(); diff --git a/ggsql-jupyter/src/results/export.rs b/ggsql-jupyter/src/results/export.rs index 9505742d5..b4a762df7 100644 --- a/ggsql-jupyter/src/results/export.rs +++ b/ggsql-jupyter/src/results/export.rs @@ -1,4 +1,4 @@ -//! A data explorer selection as CSV, TSV or HTML, for the clipboard. +//! A selection of a view as CSV, TSV or HTML. use ggsql::util::escape_html; diff --git a/ggsql-jupyter/src/results/filter.rs b/ggsql-jupyter/src/results/filter.rs index 7a27406d4..b6f1528ea 100644 --- a/ggsql-jupyter/src/results/filter.rs +++ b/ggsql-jupyter/src/results/filter.rs @@ -157,10 +157,7 @@ pub fn predicate( } Filter::IsNull { negate: false, .. } => Ok(format!("{c} IS NULL")), Filter::IsNull { negate: true, .. } => Ok(format!("{c} IS NOT NULL")), - Filter::IsEmpty { negate, .. } => { - let op = if *negate { "<>" } else { "=" }; - Ok(format!("{c} {op} {}", dialect.sql_string_literal(""))) - } + Filter::IsEmpty { negate, .. } => Ok(dialect.sql_is_empty_string(&c, *negate)), Filter::IsTrue { .. } => Ok(format!("{c} = {}", dialect.sql_boolean_literal(true))), Filter::IsFalse { .. } => Ok(format!("{c} = {}", dialect.sql_boolean_literal(false))), Filter::Text { @@ -304,8 +301,9 @@ pub fn order_terms( keys.iter() .map(|key| { let column = column_of(columns, key.column)?; - // NULLs last either way, which is where Positron's Data - // Explorer shows them. + // NULLs last either way, as the results cache's DuckDB sorts + // them, so SQL written for another database orders rows as the + // view does. Ok(dialect.sql_order_term(&dialect.quote_ident(&column.name), key.ascending, true)) }) .collect() @@ -822,7 +820,10 @@ mod tests { ])); let mysql = dialect_for_scheme("mysql").unwrap(); let sql = to_sql(&backslash, &[], "SELECT 1", &columns(), mysql).join("\n"); - assert!(sql.contains(r"`name` LIKE '%\\\\' ESCAPE '\\'"), "{sql}"); + assert!( + sql.contains(r"CAST(`name` AS BINARY) LIKE CAST('%\\\\' AS BINARY) ESCAPE '\\'"), + "{sql}" + ); } #[test] @@ -832,7 +833,10 @@ mod tests { ])); let mssql = dialect_for_scheme("mssql").unwrap(); let sql = to_sql(&filters, &[], "SELECT 1", &columns(), mssql).join("\n"); - assert!(sql.contains(r#""name" LIKE '\[a]%' ESCAPE '\'"#), "{sql}"); + assert!( + sql.contains(r#""name" COLLATE Latin1_General_CS_AS LIKE '\[a]%' ESCAPE '\'"#), + "{sql}" + ); } #[test] diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs index 8f837667a..19dd7b0a7 100644 --- a/ggsql-jupyter/src/results/mod.rs +++ b/ggsql-jupyter/src/results/mod.rs @@ -29,7 +29,7 @@ pub mod filter; pub mod methods; pub mod view; -use anyhow::{anyhow, bail, Result}; +use anyhow::{anyhow, Result}; use filter::{Filter, SortKey}; use ggsql::naming::quote_ident; use ggsql::reader::cache::{estimate_bytes, CacheTables}; @@ -42,7 +42,7 @@ use view::ResultView; /// The column holding each row's position in the query's output, added when a /// result is copied into the cache. Ties in a sort break on it, so paging is /// stable and clearing the sort restores the query's own order. -pub const ROW: &str = "__ggsql_row__"; +pub const ROW: &str = ggsql::naming::RESULT_ROW_COLUMN; /// The default byte budget for everything the results API keeps. pub const DEFAULT_MAX_BYTES: u64 = 1024 * 1024 * 1024; @@ -80,6 +80,22 @@ pub struct Column { pub repeated: bool, } +/// `bytes` as a person reads it, in the largest unit that keeps it at least 1. +fn human_bytes(bytes: u64) -> String { + const UNITS: [&str; 4] = ["bytes", "KB", "MB", "GB"]; + let mut value = bytes as f64; + let mut unit = 0; + while value >= 1024.0 && unit < UNITS.len() - 1 { + value /= 1024.0; + unit += 1; + } + if unit == 0 { + format!("{bytes} bytes") + } else { + format!("{value:.1} {}", UNITS[unit]) + } +} + /// Bytes the cached copy of a result adds per row: its `__ggsql_row__`. const ROW_BYTES: u64 = 8; @@ -165,24 +181,11 @@ impl ResultStore { } let limit = self.max_bytes - extra; let outside_cache: u64 = self.pending.iter().map(|p| p.bytes).sum::() + orders; - self.cache.evict_to(limit.saturating_sub(outside_cache))?; - self.forget_evicted()?; - while self.used()? > limit && self.pending.pop_front().is_some() {} - Ok(self.used()? <= limit) - } - - /// Forget the results the cache has evicted. - fn forget_evicted(&mut self) -> Result<()> { - let mut gone = Vec::new(); - for key in self.opened.keys() { - if self.cache.lookup(key)?.is_none() { - gone.push(key.clone()); - } - } - for key in gone { + for key in self.cache.evict_to(limit.saturating_sub(outside_cache))? { self.opened.remove(&key); } - Ok(()) + while self.used()? > limit && self.pending.pop_front().is_some() {} + Ok(self.used()? <= limit) } /// Keep `df`, produced by `query`, under a fresh key, which is returned. @@ -216,18 +219,18 @@ impl ResultStore { } fn table_name(key: &str) -> String { - format!("__ggsql_result_{key}__") + ggsql::naming::result_table(key) } /// Open a new view of the result `key`, copying it into the cache the /// first time. The result stays pinned while any view of it is open. pub fn open(&mut self, key: &str) -> Result { if let Some(i) = self.pending.iter().position(|p| p.key == key) { + // Taken out of the queue, so making room cannot drop it, and its + // copy is budgeted afresh with the row column the copy adds. If + // opening fails it goes back where it was, or last if making room + // dropped older results ahead of it. let pending = self.pending.remove(i).expect("position is in range"); - // Making room may drop older results from the front, so a result - // put back goes where it was or, past the end, last. - // Out of the queue, so making room cannot drop it; its copy is - // budgeted afresh, with the row column the copy adds. let need = pending.bytes + pending.df.height() as u64 * ROW_BYTES; let fits = match self.make_room(need) { Ok(fits) => fits, @@ -238,11 +241,7 @@ impl ResultStore { }; if !fits { self.pending.insert(i.min(self.pending.len()), pending); - bail!( - "Opening this result needs more memory than the data explorer's budget \ - has left ({} bytes). Close another Data Explorer tab and try again.", - self.max_bytes - ); + return Err(self.over_budget("Opening this result", need)); } match self.copy_into_cache(&pending) { Ok(opened) => { @@ -256,7 +255,7 @@ impl ResultStore { } } } else if self.opened.contains_key(key) { - if self.cache.lookup(key)?.is_none() { + if !self.cache.contains(key)? { self.opened.remove(key); return Err(gone()); } @@ -294,7 +293,7 @@ impl ResultStore { // Pinned from the start, so the eviction its own insertion triggers // can't take it. The budget already counts it, as a pending result. self.cache - .put(&pending.key, &pending.query, &table, df, true)?; + .put_pinned(&pending.key, &pending.query, &table, df)?; let described = self .cache @@ -366,15 +365,26 @@ impl ResultStore { let v = self.view(view)?; let need = v.max_order_bytes(); if !self.make_room(need)? { - bail!( - "Sorting or filtering this result needs more memory than the data explorer's \ - budget has left ({} bytes). Close another Data Explorer tab and try again.", - self.max_bytes - ); + return Err(self.over_budget("Sorting or filtering this result", need)); } Ok(()) } + /// The error for `what` needing `need` bytes that the budget cannot free, + /// since open views hold the rest of it. + fn over_budget(&self, what: &str, need: u64) -> anyhow::Error { + let orders: u64 = self.views.values().map(ResultView::order_bytes).sum(); + let held = self.cache.pinned_bytes().unwrap_or(0) + orders; + let free = self.max_bytes.saturating_sub(held); + anyhow!( + "{what} needs {} of memory, but open views hold all but {} of the {} kept for \ + results. Close a view of another result to make room.", + human_bytes(need), + human_bytes(free), + human_bytes(self.max_bytes) + ) + } + /// Set a view's sort keys, within the budget. pub fn set_sort(&mut self, view: &str, keys: Vec) -> Result<(), String> { // An order table is rebuilt whenever the view keeps one: clearing @@ -461,26 +471,40 @@ pub fn kind_of(type_name: &str) -> &'static str { if upper.ends_with(']') { return "unknown"; } - // The type itself, without parameters: `DECIMAL(18,3)` is `DECIMAL`, - // `ENUM('a', 'b')` is `ENUM`. + // The type itself, without parameters or MySQL's signedness: + // `DECIMAL(18,3)` is `DECIMAL`, `ENUM('a', 'b')` is `ENUM`, `INT UNSIGNED` + // is `INT`, and `TIMESTAMP(6) WITH TIME ZONE` is `TIMESTAMP`. let base = upper.split('(').next().unwrap_or("").trim(); + let base = base + .strip_suffix(" UNSIGNED") + .or_else(|| base.strip_suffix(" SIGNED")) + .unwrap_or(base); match base { - "TINYINT" | "SMALLINT" | "INTEGER" | "BIGINT" | "HUGEINT" | "UTINYINT" | "USMALLINT" - | "UINTEGER" | "UBIGINT" | "UHUGEINT" | "INT" | "INT1" | "INT2" | "INT4" | "INT8" => { - "integer" - } - "FLOAT" | "REAL" | "DOUBLE" | "DECIMAL" | "NUMERIC" => "floating", + "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INTEGER" | "BIGINT" | "HUGEINT" | "UTINYINT" + | "USMALLINT" | "UINTEGER" | "UBIGINT" | "UHUGEINT" | "INT" | "INT1" | "INT2" | "INT4" + | "INT8" | "INT64" | "SERIAL" | "BIGSERIAL" | "SMALLSERIAL" => "integer", + "FLOAT" | "FLOAT4" | "FLOAT8" | "FLOAT64" | "REAL" | "DOUBLE" | "DOUBLE PRECISION" + | "DECIMAL" | "NUMERIC" | "BIGNUMERIC" | "NUMBER" => "floating", "BOOLEAN" | "BOOL" => "boolean", "DATE" => "date", "TIMESTAMP" | "TIMESTAMP WITH TIME ZONE" + | "TIMESTAMP WITHOUT TIME ZONE" | "TIMESTAMPTZ" | "TIMESTAMP_S" | "TIMESTAMP_MS" | "TIMESTAMP_NS" - | "DATETIME" => "datetime", - "TIME" | "TIME WITH TIME ZONE" | "TIMETZ" => "time", - "VARCHAR" | "CHAR" | "BPCHAR" | "TEXT" | "STRING" | "UUID" | "ENUM" => "string", + | "TIMESTAMP_NTZ" + | "TIMESTAMP_LTZ" + | "TIMESTAMP_TZ" + | "DATETIME" + | "DATETIME2" + | "SMALLDATETIME" + | "DATETIMEOFFSET" => "datetime", + "TIME" | "TIME WITH TIME ZONE" | "TIME WITHOUT TIME ZONE" | "TIMETZ" => "time", + "VARCHAR" | "VARCHAR2" | "NVARCHAR" | "NVARCHAR2" | "CHAR" | "NCHAR" | "BPCHAR" + | "CHARACTER" | "CHARACTER VARYING" | "TEXT" | "TINYTEXT" | "MEDIUMTEXT" | "LONGTEXT" + | "NTEXT" | "CLOB" | "NCLOB" | "STRING" | "UUID" | "ENUM" => "string", _ => "unknown", } } @@ -673,7 +697,7 @@ mod tests { String::new(), ); // Break the cache's bookkeeping, so copying the result in fails. - store.cache.lookup("anything").unwrap(); + store.cache.contains("anything").unwrap(); store .cache .backend() @@ -1040,7 +1064,7 @@ mod tests { .backend() .execute_sql(&format!( "CREATE TEMP VIEW {} AS SELECT 1 AS x", - quote_ident(&format!("__ggsql_view_{view}_1__")) + quote_ident(&ggsql::naming::result_view_table(&view, 1)) )) .unwrap(); assert!(store.set_filters(&view, gt("1")).is_err()); @@ -1100,6 +1124,47 @@ mod tests { .contains("at most")); } + #[test] + fn test_an_export_with_no_rows_still_checks_its_columns() { + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(3), "q".into(), String::new()); + let id = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&id).unwrap(); + let none = Rows::Indices { indices: vec![] }; + let err = v + .export(cache.backend(), Some(&none), &[999], "csv") + .unwrap_err(); + assert!(err.contains("no column 999"), "{err}"); + } + + #[test] + fn test_profile_requests_are_bounded() { + let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); + let key = keep(&mut store, numbers(3), "q".into(), String::new()); + let id = store.open(&key).unwrap().view; + let (v, cache) = store.view_mut(&id).unwrap(); + let request = |json: serde_json::Value| -> view::ProfileRequest { + serde_json::from_value(json).unwrap() + }; + for (json, says) in [ + ( + serde_json::json!({"frequency": {"limit": 1_000_000}}), + "values", + ), + ( + serde_json::json!({"histogram": {"quantiles": vec![0.5; 1000]}}), + "quantiles", + ), + ( + serde_json::json!({"histogram": {"bins": 1_000_000}}), + "bins", + ), + ] { + let err = v.profile(cache.backend(), 0, &request(json)).unwrap_err(); + assert!(err.contains(says), "{err}"); + } + } + #[test] fn test_exports_are_bounded_more_loosely_than_pages() { let mut store = ResultStore::new(DEFAULT_MAX_BYTES).unwrap(); @@ -1120,7 +1185,7 @@ mod tests { assert_eq!(csv.lines().count(), view::MAX_PAGE_ROWS + 2); // A whole column of a longer result is refused, not built. let err = v.export(cache.backend(), None, &[0], "csv").unwrap_err(); - assert!(err.contains("copied or exported"), "{err}"); + assert!(err.contains("exported at once"), "{err}"); let all = Rows::Range { first: 0, last: usize::MAX, @@ -1128,7 +1193,7 @@ mod tests { let err = v .export(cache.backend(), Some(&all), &[0], "csv") .unwrap_err(); - assert!(err.contains("copied or exported"), "{err}"); + assert!(err.contains("exported at once"), "{err}"); } #[test] @@ -1222,5 +1287,11 @@ mod tests { assert_eq!(kind_of("TIMESTAMP_NS"), "datetime"); assert_eq!(kind_of("TIME WITH TIME ZONE"), "time"); assert_eq!(kind_of("BIGINT[]"), "unknown"); + // Other backends' names, as the Connections pane reports them. + assert_eq!(kind_of("int unsigned"), "integer"); + assert_eq!(kind_of("DOUBLE PRECISION"), "floating"); + assert_eq!(kind_of("TIMESTAMP(6) WITH TIME ZONE"), "datetime"); + assert_eq!(kind_of("character varying(255)"), "string"); + assert_eq!(kind_of("DATETIME2"), "datetime"); } } diff --git a/ggsql-jupyter/src/results/view.rs b/ggsql-jupyter/src/results/view.rs index 8cf56d3e6..1a0194a3e 100644 --- a/ggsql-jupyter/src/results/view.rs +++ b/ggsql-jupyter/src/results/view.rs @@ -12,7 +12,8 @@ use super::filter::{self, Filter, SortKey}; use super::{export, Column, ROW}; -use ggsql::naming::quote_ident; +use arrow::array::Array; +use ggsql::naming::{self, quote_ident}; use ggsql::reader::Reader; use ggsql::sql::FromItem; use ggsql::DataFrame; @@ -20,7 +21,7 @@ use serde::Deserialize; use serde_json::{json, Value}; use std::collections::HashMap; -/// The most rows one page may ask for. The Data Explorer asks for a screenful. +/// The most rows one page may ask for: many screenfuls of any grid. pub const MAX_PAGE_ROWS: usize = 10_000; /// The most rows one export may hold. An export is built as text on the @@ -48,8 +49,8 @@ impl Purpose { match self { Purpose::Page => format!("A page can hold at most {MAX_PAGE_ROWS} rows, not {n}"), Purpose::Export => format!( - "At most {MAX_EXPORT_ROWS} rows can be copied or exported at once, not {n}. \ - Select fewer rows, or use Convert to Code to run the query yourself." + "At most {MAX_EXPORT_ROWS} rows can be exported at once, not {n}. \ + Select fewer rows." ), } } @@ -108,6 +109,15 @@ fn default_limit() -> usize { 8 } +/// The most values a frequency table may ask for. +pub const MAX_FREQUENCY_VALUES: usize = 1_000; + +/// The most quantiles a histogram may ask for: each is a query of its own. +pub const MAX_QUANTILES: usize = 100; + +/// The most bins a histogram may ask for. +pub const MAX_HISTOGRAM_BINS: usize = 10_000; + /// A view of a result table. pub struct ResultView { /// The result this views. @@ -172,8 +182,10 @@ impl ResultView { match &self.predicate { None => self.table.clone(), Some(p) => format!( - "(SELECT * FROM {} WHERE {}) AS \"__ggsql_filtered__\"", - self.table, p + "(SELECT * FROM {} WHERE {}) AS {}", + self.table, + p, + quote_ident(naming::TABLE_ALIAS) ), } } @@ -263,7 +275,7 @@ impl ResultView { self.table, row = quote_ident(ROW), ); - let name = format!("__ggsql_view_{}_{}__", self.id, self.built); + let name = naming::result_view_table(&self.id, self.built as u64); let built = (|| { for stmt in dialect.create_or_replace_temp_table_sql(&name, &[], &body) { reader.execute_sql(&stmt).map_err(|e| e.to_string())?; @@ -335,9 +347,12 @@ impl ResultView { format!(" WHERE {pos} IN ({})", list.join(", ")) } }; - let select: Vec = std::iter::once(format!("{pos} AS \"__ggsql_pos__\"")) - .chain(names) - .collect(); + let select: Vec = std::iter::once(format!( + "{pos} AS {}", + quote_ident(naming::VIEW_POSITION_COLUMN) + )) + .chain(names) + .collect(); let sql = format!( "SELECT {} FROM {from}{filter} ORDER BY {pos}", select.join(", ") @@ -385,14 +400,13 @@ impl ResultView { unique.sort_unstable(); unique.dedup(); let df = self.fetch(reader, cols, Fetch::Positions(&unique))?; - let pos = df.column("__ggsql_pos__").map_err(|e| e.to_string())?; + let pos = df + .column(naming::VIEW_POSITION_COLUMN) + .map_err(|e| e.to_string())?; + let positions = ints(pos).ok_or("The view's positions are not integers")?; let at: HashMap = (0..df.height()) - .filter_map(|r| { - ggsql::array_util::value_to_string(pos, r) - .parse::() - .ok() - .map(|p| (p, r)) - }) + .filter(|&r| !positions.is_null(r)) + .map(|r| (positions.value(r) as usize, r)) .collect(); let picked = indices .iter() @@ -436,6 +450,16 @@ impl ResultView { cols: &[usize], format: &str, ) -> Result { + // Checked first: the header names every column, even with no rows. + let header = cols + .iter() + .map(|&i| { + self.columns + .get(i) + .map(|c| c.name.clone()) + .ok_or_else(|| format!("There is no column {i}")) + }) + .collect::, _>>()?; let (df, picked) = match rows { None => { if self.selected_rows > MAX_EXPORT_ROWS { @@ -456,7 +480,6 @@ impl ResultView { .collect() }) .collect(); - let header: Vec = cols.iter().map(|&i| self.columns[i].name.clone()).collect(); export::format(Some(&header), &cells, format) } @@ -493,6 +516,28 @@ impl ResultView { .columns .get(index) .ok_or_else(|| format!("There is no column {index}"))?; + if let Some(f) = &request.frequency { + if f.limit > MAX_FREQUENCY_VALUES { + return Err(format!( + "A frequency table can hold at most {MAX_FREQUENCY_VALUES} values, not {}", + f.limit + )); + } + } + if let Some(h) = &request.histogram { + if h.quantiles.len() > MAX_QUANTILES { + return Err(format!( + "A histogram can ask for at most {MAX_QUANTILES} quantiles, not {}", + h.quantiles.len() + )); + } + if h.bins > MAX_HISTOGRAM_BINS { + return Err(format!( + "A histogram can have at most {MAX_HISTOGRAM_BINS} bins, not {}", + h.bins + )); + } + } let mut out = json!({}); let kind = col.kind.as_str(); @@ -565,7 +610,7 @@ impl ResultView { let sql = format!("SELECT {} FROM {}", parts.join(", "), self.relation()); let df = reader.execute_sql(&sql).map_err(|e| e.to_string())?; let get = |name: &str| first_string(&df, name); - let get_i64 = |name: &str| get(name).and_then(|s| s.parse::().ok()); + let get_i64 = |name: &str| df.column(name).ok().and_then(first_int); let mut out = json!({}); match col.kind.as_str() { @@ -620,9 +665,11 @@ impl ResultView { self.relation() ); let df = reader.execute_sql(&sql).ok()?; - let sd = first_string(&df, "sd")?; - sd.parse::().ok().filter(|v| v.is_finite())?; - Some(sd) + // Reported as the database writes it, which keeps its precision; the + // number is read only to leave out a NaN or infinity. + let sd_col = df.column("sd").ok()?; + floats(sd_col).filter(|f| !f.is_empty() && !f.is_null(0) && f.value(0).is_finite())?; + first_string(&df, "sd") } /// The `q` quantile of a column, through the dialect's quantile hook. @@ -633,7 +680,10 @@ impl ResultView { let expr = reader .dialect() .sql_quantile(&col.name, q, FromItem::Query(&from), &[]); - let sql = format!("SELECT {expr} AS \"q\" FROM ({from}) AS \"__ggsql_qt__\""); + let sql = format!( + "SELECT {expr} AS \"q\" FROM ({from}) AS {}", + quote_ident(naming::TABLE_ALIAS) + ); let df = reader.execute_sql(&sql).ok()?; first_string(&df, "q") } @@ -662,11 +712,19 @@ impl ResultView { self.relation() )) .ok()?; - let min: f64 = first_string(&bounds, "min")?.parse().ok()?; - let max: f64 = first_string(&bounds, "max")?.parse().ok()?; - let count: usize = first_string(&bounds, "cnt") - .and_then(|s| s.parse().ok()) - .unwrap_or(0); + let first_float = |name: &str| { + floats(bounds.column(name).ok()?) + .filter(|f| !f.is_empty() && !f.is_null(0)) + .map(|f| f.value(0)) + }; + let min = first_float("min")?; + let max = first_float("max")?; + let count = bounds + .column("cnt") + .ok() + .and_then(first_int) + .unwrap_or(0) + .max(0) as usize; let quantiles: Vec = req .quantiles @@ -700,23 +758,22 @@ impl ResultView { let sql = format!( "SELECT CASE WHEN \"bin\" >= {bins} THEN {last} ELSE \"bin\" END AS \"b\", COUNT(*) AS \"n\" \ FROM (SELECT FLOOR((CAST({c} AS {float}) - {min}) / {width}) AS \"bin\" \ - FROM {} WHERE {c} IS NOT NULL) AS \"__bins__\" \ + FROM {} WHERE {c} IS NOT NULL) AS {alias} \ GROUP BY \"b\" ORDER BY \"b\"", self.relation(), last = bins - 1, + alias = quote_ident(naming::TABLE_ALIAS), ); let df = reader.execute_sql(&sql).ok()?; let mut counts = vec![0i64; bins]; - let (b, n) = (df.column("b").ok()?, df.column("n").ok()?); + let (b, n) = (floats(df.column("b").ok()?)?, ints(df.column("n").ok()?)?); for i in 0..df.height() { - let bin = ggsql::array_util::value_to_string(b, i) - .parse::() - .ok() - .map(|v| v as usize); - if let Some(bin) = bin.filter(|&bin| bin < bins) { - counts[bin] = ggsql::array_util::value_to_string(n, i) - .parse() - .unwrap_or(0); + if b.is_null(i) || n.is_null(i) { + continue; + } + let bin = b.value(i) as usize; + if bin < bins { + counts[bin] = n.value(i); } } let edges: Vec = (0..=bins) @@ -735,15 +792,9 @@ impl ResultView { self.relation() )) .ok()?; - let (v, n) = (df.column("value").ok()?, df.column("count").ok()?); + let (v, n) = (df.column("value").ok()?, ints(df.column("count").ok()?)?); let values: Vec = (0..df.height()).map(|i| cell_string(v, i)).collect(); - let counts: Vec = (0..df.height()) - .map(|i| { - ggsql::array_util::value_to_string(n, i) - .parse() - .unwrap_or(0) - }) - .collect(); + let counts: Vec = n.iter().map(|c| c.unwrap_or(0)).collect(); let total = scalar_i64( reader, &format!("SELECT COUNT({c}) AS \"n\" FROM {}", self.relation()), @@ -776,14 +827,30 @@ fn scalar_i64(reader: &dyn Reader, sql: &str) -> Result { .into_iter() .next() .ok_or("The query returned no columns")?; - Ok(first_string(&df, &name) - .and_then(|s| s.parse().ok()) - .unwrap_or(0)) + Ok(df.column(&name).ok().and_then(first_int).unwrap_or(0)) +} + +/// A numeric column as 64-bit integers, or `None` when it cannot be one. +fn ints(col: &arrow::array::ArrayRef) -> Option { + let cast = arrow::compute::cast(col, &arrow::datatypes::DataType::Int64).ok()?; + ggsql::array_util::as_i64(&cast).ok().cloned() +} + +/// A numeric column as 64-bit floats, or `None` when it cannot be one. +fn floats(col: &arrow::array::ArrayRef) -> Option { + let cast = arrow::compute::cast(col, &arrow::datatypes::DataType::Float64).ok()?; + ggsql::array_util::as_f64(&cast).ok().cloned() +} + +/// Row 0 of a numeric column as an integer, or `None` when absent or null. +fn first_int(col: &arrow::array::ArrayRef) -> Option { + ints(col) + .filter(|a| !a.is_empty() && !a.is_null(0)) + .map(|a| a.value(0)) } /// Row 0 of `name` as text, or `None` when absent or null. fn first_string(df: &DataFrame, name: &str) -> Option { - use arrow::array::Array; let c = df.column(name).ok()?; if c.is_empty() || c.is_null(0) { None @@ -794,16 +861,14 @@ fn first_string(df: &DataFrame, name: &str) -> Option { /// A cell as text, with a null as empty. fn cell_string(col: &arrow::array::ArrayRef, row: usize) -> String { - use arrow::array::Array; if col.is_null(row) { return String::new(); } - crate::display::cell_text(col, row) + ggsql::array_util::value_to_string(col, row) } /// A cell for a page: its text, or `null`. fn cell_value(col: &arrow::array::ArrayRef, row: usize) -> Value { - use arrow::array::Array; if col.is_null(row) { Value::Null } else { diff --git a/ggsql-jupyter/tests/test_integration.py b/ggsql-jupyter/tests/test_integration.py index 4db7da852..b14003784 100644 --- a/ggsql-jupyter/tests/test_integration.py +++ b/ggsql-jupyter/tests/test_integration.py @@ -747,7 +747,7 @@ def test_the_environment_sets_the_rows_a_console_shows(self, kernel_binary): with _launch_kernel( kernel_binary, ["--session-mode", "console"], - env={"GGSQL_CONSOLE_ROWS": "3"}, + env={"GGSQL_CONSOLE_ROWS": "3", "GGSQL_KEEP_RESULTS": "1"}, ) as km: kc = km.client() kc.start_channels() diff --git a/ggsql-vscode/AGENTS.md b/ggsql-vscode/AGENTS.md index f569ca6b1..97b7b629f 100644 --- a/ggsql-vscode/AGENTS.md +++ b/ggsql-vscode/AGENTS.md @@ -103,7 +103,7 @@ The extension declares `contributes.languageRuntimes` for `ggsql` (see `package. Both live in `manager.ts`, as does one piece of user configuration: **`ggsql.consoleRows` reaches the kernel as `GGSQL_CONSOLE_ROWS` in the spec's `env`, not as `--console-rows`.** A kernel found on the machine may predate the option, and it ignores an unknown variable where it would refuse to start on an unknown flag. `consoleRowsEnv` leaves out a value the kernel would reject — below 1, or above `u32::MAX`, which the setting's `maximum` also states — for the same reason. Like the session mode, it is fixed when a session starts. -The same goes for **`GGSQL_KEEP_RESULTS`**, which `keepResultsEnv` sets for a console session when this Positron has extension-hosted Data Explorer backends: the kernel keeps query results only when asked, so a session whose results nothing can open holds none. +The same goes for **`GGSQL_KEEP_RESULTS`**, which `keepResultsEnv` sets for a console session when this Positron has extension-hosted Data Explorer backends: the kernel keeps query results only when asked, so a session whose results nothing can open holds none. With it goes **`GGSQL_RESULTS_MAX_BYTES`** from the `ggsql.resultsMemory` setting, through `resultsMemoryEnv`, which leaves out anything the kernel's byte-count parser would refuse. The kernel side of the contract — why the comm alone creates the pane entry, and why an output bundle alongside it would show the plot twice — is in [`/ggsql-jupyter/AGENTS.md`](../ggsql-jupyter/AGENTS.md). @@ -182,7 +182,8 @@ The one case that interrupts the user is the dead end: nothing runnable anywhere ```json { "ggsql.kernelPath": "string", // an extra kernel to offer, listed first - "ggsql.consoleRows": 10 // rows of a long result a console shows + "ggsql.consoleRows": 10, // rows of a long kept result a console shows + "ggsql.resultsMemory": "1gb" // memory a console may keep results in } ``` diff --git a/ggsql-vscode/package.json b/ggsql-vscode/package.json index d7e50becd..6c4fd5cc3 100644 --- a/ggsql-vscode/package.json +++ b/ggsql-vscode/package.json @@ -170,7 +170,13 @@ "default": 10, "minimum": 1, "maximum": 4294967295, - "markdownDescription": "How many rows of a query result the ggsql console shows. A result with up to twice this many rows is shown in full; a longer one shows this many, then a count of the rest. Every result has a link beneath it that opens the whole result in the Data Explorer. Notebooks are unaffected. Takes effect in consoles started after the change." + "markdownDescription": "How many rows of a query result the ggsql console shows. A result with up to twice this many rows is shown in full; a longer one shows this many, then a count of the rest. A result kept for the Data Explorer has a link beneath it that opens the whole of it; a result that could not be kept is shown up to 100 rows. Notebooks are unaffected. Takes effect in consoles started after the change." + }, + "ggsql.resultsMemory": { + "type": "string", + "default": "1gb", + "pattern": "^\\s*[0-9]{1,12}\\s*([kKmMgG][bB])?\\s*$", + "markdownDescription": "How much memory a ggsql console may use to keep query results for the Data Explorer, such as `512mb` or `2gb`. Results open in a Data Explorer are always kept; older ones make way for new ones. Takes effect in consoles started after the change." }, "ggsql.enableSqlFiles": { "type": "boolean", diff --git a/ggsql-vscode/src/manager.ts b/ggsql-vscode/src/manager.ts index e6bce461e..2102c37ac 100644 --- a/ggsql-vscode/src/manager.ts +++ b/ggsql-vscode/src/manager.ts @@ -552,6 +552,19 @@ export function consoleRowsEnv(rows: unknown): Record { /** The most rows the kernel accepts for `--console-rows`, a `u32`. */ export const MAX_CONSOLE_ROWS = 4294967295; +/** + * The kernel environment for the `ggsql.resultsMemory` setting, a byte count + * such as `512mb`. As for `consoleRows`, a value the kernel would reject is + * left out, so it uses its default rather than refusing to start. + */ +export function resultsMemoryEnv(value: unknown): Record { + if (typeof value !== 'string') { + return {}; + } + const match = /^\s*([0-9]{1,12})\s*([kmg]b)?\s*$/i.exec(value); + return match && Number(match[1]) > 0 ? { GGSQL_RESULTS_MAX_BYTES: value.trim() } : {}; +} + /** * The kernel environment that turns on keeping query results, which the * kernel only does when asked. A console session whose results this @@ -600,6 +613,7 @@ function createKernelSpec( RUST_LOG: 'error', ...consoleRowsEnv(vscode.workspace.getConfiguration('ggsql').get('consoleRows')), ...keepResultsEnv(keepResults), + ...(keepResults ? resultsMemoryEnv(vscode.workspace.getConfiguration('ggsql').get('resultsMemory')) : {}), }, kernel_protocol_version: '5.3', }; diff --git a/ggsql-vscode/src/test/manager.test.ts b/ggsql-vscode/src/test/manager.test.ts index 8805ab92b..078f7bb11 100644 --- a/ggsql-vscode/src/test/manager.test.ts +++ b/ggsql-vscode/src/test/manager.test.ts @@ -1,6 +1,6 @@ import * as assert from 'assert'; import * as vscode from 'vscode'; -import { GgsqlRuntimeManager, MAX_CONSOLE_ROWS, consoleRowsEnv, createDynState, getSupervisorApi, keepResultsEnv } from '../manager'; +import { GgsqlRuntimeManager, MAX_CONSOLE_ROWS, consoleRowsEnv, createDynState, getSupervisorApi, keepResultsEnv, resultsMemoryEnv } from '../manager'; suite('manager', () => { // The suites run in stock VS Code, where positron.positron-supervisor is @@ -13,6 +13,18 @@ suite('manager', () => { ); }); + test('resultsMemoryEnv passes a byte count the kernel accepts', () => { + assert.deepStrictEqual(resultsMemoryEnv(' 512MB '), { GGSQL_RESULTS_MAX_BYTES: '512MB' }); + assert.deepStrictEqual(resultsMemoryEnv('2gb'), { GGSQL_RESULTS_MAX_BYTES: '2gb' }); + assert.deepStrictEqual(resultsMemoryEnv('1048576'), { GGSQL_RESULTS_MAX_BYTES: '1048576' }); + }); + + test('resultsMemoryEnv leaves out a value the kernel would refuse', () => { + for (const value of ['', '0', '0gb', '1.5gb', '2 terabytes', '-1mb', '1234567890123gb', 512, undefined]) { + assert.deepStrictEqual(resultsMemoryEnv(value), {}, String(value)); + } + }); + test('consoleRowsEnv passes a row count the kernel accepts', () => { assert.deepStrictEqual(consoleRowsEnv(25), { GGSQL_CONSOLE_ROWS: '25' }); }); diff --git a/ggsql-vscode/src/test/resultLinks.test.ts b/ggsql-vscode/src/test/resultLinks.test.ts index 336aaeb43..12d8b43bd 100644 --- a/ggsql-vscode/src/test/resultLinks.test.ts +++ b/ggsql-vscode/src/test/resultLinks.test.ts @@ -12,10 +12,9 @@ suite('parseResultLink', () => { ); }); - test('ignores other schemes, the old one included', () => { + test('ignores other schemes', () => { assert.strictEqual(parseResultLink('x-r-help:topic'), undefined); assert.strictEqual(parseResultLink('https://ggsql.org'), undefined); - assert.strictEqual(parseResultLink('x-ggsql-view:s/abc'), undefined); }); test('rejects a key the kernel could not have produced', () => { diff --git a/src/AGENTS.md b/src/AGENTS.md index 052bded6f..2f4d92514 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -59,7 +59,7 @@ Grammar lives in [`/tree-sitter-ggsql/`](../tree-sitter-ggsql/) — when adding `SqlDialect` trait in `mod.rs` lets each driver supply its own type names, information-schema queries, literal and pattern syntax (`quote_ident`, `sql_string_literal`, `sql_like`, `sql_regex_match`, `sql_order_term`), and spatial helper methods (`sql_st_transform`, `sql_geometry_to_wkb`, `sql_geometry_bbox`, `sql_ensure_geometry`, `sql_select_replace`, `sql_spatial_setup`). -**Caching layer.** `CachingReader` (`cache.rs`) wraps a primary reader plus an in-memory `CacheBackend`, splitting work across two `Reader` surfaces. **`execute_sql` = source**: base reads of the user's data plus user setup/DML run on the primary (with result memoization), except `ggsql:` builtins, the `__ggsql_cache_meta__` table, and reads that reference a cache-resident internal table, which go to the cache. **`execute_sql_cached` = compute**: all dialect-generated/derived SQL (schema probes, stats, projection/map transforms, spatial setup, final layer queries — everything operating on `__ggsql_*` tables) runs on the cache; it defaults to `execute_sql` so a plain reader runs everything on one connection. Cache routing is by **exact-identifier membership** in the set of tables registered into the cache. Memoization keys on `hash(primary_uri + sql)` and is tracked in the `__ggsql_cache_meta__` table inside the cache backend. Each memoized read is bounded by a **TTL** (default 300s) and the whole memo by an **LRU byte budget** (default 512 MB); both are configurable via `CacheConfig` (env `GGSQL_CACHE_DISABLED`/`GGSQL_CACHE_TTL`/`GGSQL_CACHE_MAX_BYTES`, or per-connection URI query parameters `?cache_ttl=…&cache_max_bytes=…&cache_disabled=…`). The `__ggsql_cache_meta__` table is queryable for introspection (`SELECT * FROM __ggsql_cache_meta__`). Pure/non-visual SQL (CLI table fallback, Jupyter) goes through `execute_sql` so it reads the primary rather than the empty cache. `Reader::materialize_table` (default = `CREATE TEMP TABLE` on the reader, no Rust roundtrip) is overridden to read the body via the source surface and `register()` the result into the cache, so the primary is never written to; `Reader::caches_sources()` (default `false`, `true` for `CachingReader`) gates the executor's per-layer source staging: file sources are staged on the cache surface, while identifiers go through `materialize_table`, which routes the read to the cache (CTEs, builtins, cache-resident tables) or the primary as needed. `dialect()` returns the **cache** dialect, and every compute-surface failure is prefixed with the cache backend's scheme (``on the `duckdb` cache backend: …``) so a cache-dialect driver error is not mistaken for one from the user's own connection. Selected via the composite `+://` scheme (`reader_from_uri` / `split_cache_uri`) or the CLI `--cache` flag; off by default. The memo's bookkeeping — registering a table, its row in `__ggsql_cache_meta__`, LRU eviction under the byte budget — is `CacheTables`, which other callers use on a backend of their own (the Jupyter kernel's results API does, without wrapping the user's reader). An entry can be **pinned**, exempting it from eviction until unpinned (pins are kept in memory, not in the metadata table); `CachingReader` never pins, and the TTL and enabled checks stay in `CachingReader`. `CacheTables` takes a byte budget rather than a `CacheConfig`, and `evict_to`/`total_bytes` let a caller with a wider budget evict to that instead. +**Caching layer.** `CachingReader` (`cache.rs`) wraps a primary reader plus an in-memory `CacheBackend`, splitting work across two `Reader` surfaces. **`execute_sql` = source**: base reads of the user's data plus user setup/DML run on the primary (with result memoization), except `ggsql:` builtins, the `__ggsql_cache_meta__` table, and reads that reference a cache-resident internal table, which go to the cache. **`execute_sql_cached` = compute**: all dialect-generated/derived SQL (schema probes, stats, projection/map transforms, spatial setup, final layer queries — everything operating on `__ggsql_*` tables) runs on the cache; it defaults to `execute_sql` so a plain reader runs everything on one connection. Cache routing is by **exact-identifier membership** in the set of tables registered into the cache. Memoization keys on `hash(primary_uri + sql)` and is tracked in the `__ggsql_cache_meta__` table inside the cache backend. Each memoized read is bounded by a **TTL** (default 300s) and the whole memo by an **LRU byte budget** (default 512 MB); both are configurable via `CacheConfig` (env `GGSQL_CACHE_DISABLED`/`GGSQL_CACHE_TTL`/`GGSQL_CACHE_MAX_BYTES`, or per-connection URI query parameters `?cache_ttl=…&cache_max_bytes=…&cache_disabled=…`). The `__ggsql_cache_meta__` table is queryable for introspection (`SELECT * FROM __ggsql_cache_meta__`). Pure/non-visual SQL (CLI table fallback, Jupyter) goes through `execute_sql` so it reads the primary rather than the empty cache. `Reader::materialize_table` (default = `CREATE TEMP TABLE` on the reader, no Rust roundtrip) is overridden to read the body via the source surface and `register()` the result into the cache, so the primary is never written to; `Reader::caches_sources()` (default `false`, `true` for `CachingReader`) gates the executor's per-layer source staging: file sources are staged on the cache surface, while identifiers go through `materialize_table`, which routes the read to the cache (CTEs, builtins, cache-resident tables) or the primary as needed. `dialect()` returns the **cache** dialect, and every compute-surface failure is prefixed with the cache backend's scheme (``on the `duckdb` cache backend: …``) so a cache-dialect driver error is not mistaken for one from the user's own connection. Selected via the composite `+://` scheme (`reader_from_uri` / `split_cache_uri`) or the CLI `--cache` flag; off by default. The memo's bookkeeping — registering a table, its row in `__ggsql_cache_meta__`, LRU eviction under the byte budget — is `CacheTables`, which other callers use on a backend of their own (the Jupyter kernel's results API does, without wrapping the user's reader). An entry can be **pinned**, exempting it from eviction until unpinned (pins are kept in memory, not in the metadata table); `CachingReader` never pins, and the TTL and enabled checks stay in `CachingReader`. `CacheTables` takes a byte budget rather than a `CacheConfig`; `put` and `put_pinned` add entries, `contains` checks for one, and `evict_to`/`total_bytes`/`pinned_bytes` let a caller with a wider budget evict to that instead, `evict_to` returning the keys it evicted. Only an existing entry can be pinned. ### `execute/` diff --git a/src/array_util.rs b/src/array_util.rs index 71bab0fde..5d76a7bea 100644 --- a/src/array_util.rs +++ b/src/array_util.rs @@ -193,12 +193,27 @@ pub fn value_to_display(array: &ArrayRef, idx: usize) -> Option { .value(idx) .to_string(), DataType::Boolean => as_bool(array).unwrap().value(idx).to_string(), - _ => arrow::util::display::ArrayFormatter::try_new(array.as_ref(), &Default::default()) - .map(|f| f.value(idx).to_string()) - .unwrap_or_else(|_| format!("{:?}", array.data_type())), + // Arrow here reads only fixed offsets as zones, not names such as + // `UTC`. The stored value is an instant, so retagging the cell as + // `+00:00` writes the same moment, in UTC. + DataType::Timestamp(unit, Some(_)) => arrow::compute::cast( + &array.slice(idx, 1), + &DataType::Timestamp(*unit, Some("+00:00".into())), + ) + .ok() + .and_then(|cell| formatted(&cell, 0)) + .unwrap_or_else(|| format!("{:?}", array.data_type())), + _ => formatted(array, idx).unwrap_or_else(|| format!("{:?}", array.data_type())), }) } +/// Arrow's own display text for the value at `idx`. +fn formatted(array: &ArrayRef, idx: usize) -> Option { + arrow::util::display::ArrayFormatter::try_new(array.as_ref(), &Default::default()) + .ok() + .map(|f| f.value(idx).to_string()) +} + /// Get a string representation of a value at an index, for any array type. /// Nulls render as the literal "null". pub fn value_to_string(array: &ArrayRef, idx: usize) -> String { @@ -244,6 +259,18 @@ pub fn cast_to_strings(array: &ArrayRef, null_as: &str) -> Result> { mod tests { use super::*; + #[test] + fn test_timestamps_with_a_time_zone_are_written_in_utc() { + let arr: ArrayRef = Arc::new( + arrow::array::TimestampMicrosecondArray::from(vec![Some(1_704_164_645_000_000), None]) + .with_timezone("UTC"), + ); + assert_eq!(value_to_string(&arr, 0), "2024-01-02T03:04:05Z"); + assert_eq!(value_to_display(&arr, 1), None); + let dates: ArrayRef = Arc::new(arrow::array::Date32Array::from(vec![19723])); + assert_eq!(value_to_string(&dates, 0), "2024-01-01"); + } + #[test] fn test_downcast_f64() { let arr: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0])); diff --git a/src/naming.rs b/src/naming.rs index acc716b64..75f34fbc8 100644 --- a/src/naming.rs +++ b/src/naming.rs @@ -94,6 +94,13 @@ pub const SCHEMA_ALIAS: &str = concatcp!(GGSQL_SUFFIX, "schema", GGSQL_SUFFIX); /// Alias for the derived table a TABULATE selection wraps its source in pub const TABLE_ALIAS: &str = concatcp!(GGSQL_SUFFIX, "table", GGSQL_SUFFIX); +/// Column holding each row's position in a query's output, in a result kept +/// for the Jupyter kernel's results API. +pub const RESULT_ROW_COLUMN: &str = concatcp!(GGSQL_PREFIX, "row", GGSQL_SUFFIX); + +/// Column holding each row's position in a view of a kept result. +pub const VIEW_POSITION_COLUMN: &str = concatcp!(GGSQL_PREFIX, "pos", GGSQL_SUFFIX); + /// Column name for densification group ID. /// Used by geoms that expand geometry (tile, segment, ribbon, rule) to group /// densified vertices back into their original feature. @@ -165,6 +172,32 @@ pub fn cache_result_table(key: &str) -> String { ) } +/// Table name for a query result kept by the Jupyter kernel's results API. +/// +/// Format: `__ggsql_result___` +/// +/// # Example +/// ``` +/// use ggsql::naming; +/// assert_eq!(naming::result_table("abc"), "__ggsql_result_abc__"); +/// ``` +pub fn result_table(key: &str) -> String { + format!("{GGSQL_PREFIX}result_{key}{GGSQL_SUFFIX}") +} + +/// Table name for the `n`th order table built for a view of a kept result. +/// +/// Format: `__ggsql_view____` +/// +/// # Example +/// ``` +/// use ggsql::naming; +/// assert_eq!(naming::result_view_table("v", 2), "__ggsql_view_v_2__"); +/// ``` +pub fn result_view_table(view: &str, n: u64) -> String { + format!("{GGSQL_PREFIX}view_{view}_{n}{GGSQL_SUFFIX}") +} + /// Generate temp table name for a materialized explicit layer source. /// /// Format: `__ggsql_layer_src____` diff --git a/src/reader/cache.rs b/src/reader/cache.rs index c12e4394f..6230ea93b 100644 --- a/src/reader/cache.rs +++ b/src/reader/cache.rs @@ -104,7 +104,7 @@ pub fn parse_human_bytes(s: &str) -> Option { } else { (lower.as_str(), 1) }; - num.trim().parse::().ok().map(|n| n * mult) + num.trim().parse::().ok()?.checked_mul(mult) } /// Wall-clock milliseconds since the UNIX epoch. On a clock earlier than the @@ -127,9 +127,9 @@ pub fn estimate_bytes(df: &DataFrame) -> i64 { } /// One memo row, trimmed to the fields consulted at runtime. -pub struct MemoEntry { - pub table_name: String, - pub fetched_at_epoch_ms: i64, +pub(crate) struct MemoEntry { + pub(crate) table_name: String, + pub(crate) fetched_at_epoch_ms: i64, } /// Tables kept in an in-memory cache backend under a byte budget. @@ -189,8 +189,13 @@ impl CacheTables { Ok(()) } + /// Whether there is an entry for `key`. + pub fn contains(&self, key: &str) -> Result { + Ok(self.lookup(key)?.is_some()) + } + /// Look up the entry for `key`. - pub fn lookup(&self, key: &str) -> Result> { + pub(crate) fn lookup(&self, key: &str) -> Result> { self.ensure_meta_table()?; let sql = format!( "SELECT table_name, fetched_at_epoch_ms FROM {} WHERE cache_key = {}", @@ -210,9 +215,18 @@ impl CacheTables { } /// Keep `df` as the table `table` under `key`, labelled with the `sql` - /// that produced it, then evict over the budget. A `pinned` entry is never - /// evicted, so it survives the eviction its own insertion triggers. - pub fn put( + /// that produced it, then evict over the budget. + pub fn put(&self, key: &str, sql: &str, table: &str, df: DataFrame) -> Result<()> { + self.put_entry(key, sql, table, df, false) + } + + /// [`put`](Self::put), pinned from the start, so it survives the eviction + /// its own insertion triggers. + pub fn put_pinned(&self, key: &str, sql: &str, table: &str, df: DataFrame) -> Result<()> { + self.put_entry(key, sql, table, df, true) + } + + fn put_entry( &self, key: &str, sql: &str, @@ -235,7 +249,7 @@ impl CacheTables { } else { self.pinned.borrow_mut().remove(key); } - self.evict_to(self.max_bytes) + self.evict_to(self.max_bytes).map(drop) } /// Record an entry in the metadata table. @@ -291,15 +305,21 @@ impl CacheTables { /// Pin or unpin the entry for `key`, which also counts as a use of it. /// Pinned entries are never evicted. Unpinning evicts over the budget, - /// since the entry counts towards it again. + /// since the entry counts towards it again. There must be an entry to pin. pub fn set_pinned(&self, key: &str, pinned: bool) -> Result<()> { - self.touch(key)?; if pinned { + if !self.contains(key)? { + return Err(crate::GgsqlError::InternalError(format!( + "There is no cache entry {key:?} to pin" + ))); + } + self.touch(key)?; self.pinned.borrow_mut().insert(key.to_string()); Ok(()) } else { + self.touch(key)?; self.pinned.borrow_mut().remove(key); - self.evict_to(self.max_bytes) + self.evict_to(self.max_bytes).map(drop) } } @@ -359,7 +379,8 @@ impl CacheTables { /// Evict unpinned entries, least recently used first, until the total /// byte estimate is within `max_bytes` or nothing unpinned is left. A /// caller with a budget wider than this cache's passes what it allows. - pub fn evict_to(&self, max_bytes: u64) -> Result<()> { + pub fn evict_to(&self, max_bytes: u64) -> Result> { + let mut evicted = Vec::new(); while self.total_bytes()? > max_bytes { let pinned = self.pinned.borrow(); let exclude = if pinned.is_empty() { @@ -375,13 +396,14 @@ impl CacheTables { naming::quote_ident(naming::CACHE_META_TABLE) ))?; if df.height() == 0 { - return Ok(()); + break; } let key = as_str(df.column("cache_key")?)?.value(0).to_string(); let table = as_str(df.column("table_name")?)?.value(0).to_string(); self.drop_entry(&key, &table)?; + evicted.push(key); } - Ok(()) + Ok(evicted) } /// Drop every entry, one at a time. @@ -567,7 +589,7 @@ impl Reader for CachingReader { // Cache row-returning reads only. if super::returns_rows(sql) && df.width() > 0 { let table = naming::cache_result_table(&key); - self.tables.put(&key, sql, &table, df.clone(), false)?; + self.tables.put(&key, sql, &table, df.clone())?; } Ok(df) @@ -1452,9 +1474,9 @@ mod behavior_tests { fn test_pinned_entries_survive_eviction() { let tables = tiny_budget_tables(); let frame = || df! { "x" => vec![1_i64, 2, 3] }.unwrap(); - tables.put("a", "a", "__t_a__", frame(), true).unwrap(); - tables.put("b", "b", "__t_b__", frame(), false).unwrap(); - tables.put("c", "c", "__t_c__", frame(), true).unwrap(); + tables.put_pinned("a", "a", "__t_a__", frame()).unwrap(); + tables.put("b", "b", "__t_b__", frame()).unwrap(); + tables.put_pinned("c", "c", "__t_c__", frame()).unwrap(); // Over a 1-byte budget, only the unpinned entry goes. assert!(tables.lookup("a").unwrap().is_some()); @@ -1472,8 +1494,8 @@ mod behavior_tests { let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), u64::MAX); assert_eq!(tables.pinned_bytes().unwrap(), 0); let frame = || df! { "x" => vec![1_i64, 2, 3] }.unwrap(); - tables.put("a", "a", "__t_a__", frame(), true).unwrap(); - tables.put("b", "b", "__t_b__", frame(), false).unwrap(); + tables.put_pinned("a", "a", "__t_a__", frame()).unwrap(); + tables.put("b", "b", "__t_b__", frame()).unwrap(); let total = tables.total_bytes().unwrap(); assert!(total > 0); assert_eq!(tables.pinned_bytes().unwrap() * 2, total); @@ -1485,7 +1507,7 @@ mod behavior_tests { fn test_unpinning_makes_an_entry_evictable() { let tables = tiny_budget_tables(); let frame = || df! { "x" => vec![1_i64] }.unwrap(); - tables.put("a", "a", "__t_a__", frame(), true).unwrap(); + tables.put_pinned("a", "a", "__t_a__", frame()).unwrap(); tables.set_pinned("a", false).unwrap(); assert!(tables.lookup("a").unwrap().is_none()); assert!(tables @@ -1500,39 +1522,43 @@ mod behavior_tests { let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), 1); assert_eq!(tables.total_bytes().unwrap(), 0); tables.touch("nothing").unwrap(); - tables.set_pinned("nothing", true).unwrap(); - tables.evict_to(0).unwrap(); - assert!(tables.lookup("nothing").unwrap().is_none()); + tables.set_pinned("nothing", false).unwrap(); + assert!(tables.evict_to(0).unwrap().is_empty()); + assert!(!tables.contains("nothing").unwrap()); } #[test] - fn test_pins_stay_out_of_the_metadata_table() { - let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), u64::MAX); + fn test_only_an_entry_can_be_pinned() { + let tables = tiny_budget_tables(); + assert!(tables.set_pinned("nothing", true).is_err()); + // No pin was left behind to keep a later entry of that key. tables .put( - "a", - "a", - "__t_a__", + "nothing", + "q", + "__t_n__", df! { "x" => vec![1_i64] }.unwrap(), - true, ) .unwrap(); - let meta = tables - .backend() - .execute_sql(&format!( - "SELECT * FROM {}", - naming::quote_ident(naming::CACHE_META_TABLE) - )) - .unwrap(); - assert!(!meta.get_column_names().iter().any(|c| c == "pinned")); + assert!(!tables.contains("nothing").unwrap()); + } + + #[test] + fn test_evict_to_reports_what_it_evicted() { + let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), u64::MAX); + let frame = || df! { "x" => vec![1_i64] }.unwrap(); + tables.put("a", "a", "__t_a__", frame()).unwrap(); + tables.put_pinned("b", "b", "__t_b__", frame()).unwrap(); + assert_eq!(tables.evict_to(0).unwrap(), ["a"]); + assert!(tables.contains("b").unwrap()); } #[test] fn test_evict_to_a_caller_budget_counts_pinned_entries_but_keeps_them() { let tables = CacheTables::new(Box::new(DuckDBReader::new_in_memory().unwrap()), u64::MAX); let frame = || df! { "x" => vec![1_i64, 2, 3] }.unwrap(); - tables.put("a", "a", "__t_a__", frame(), true).unwrap(); - tables.put("b", "b", "__t_b__", frame(), false).unwrap(); + tables.put_pinned("a", "a", "__t_a__", frame()).unwrap(); + tables.put("b", "b", "__t_b__", frame()).unwrap(); let both = tables.total_bytes().unwrap(); assert!(both > 0); diff --git a/src/reader/dialects/mod.rs b/src/reader/dialects/mod.rs index dc6c91ed3..057df7584 100644 --- a/src/reader/dialects/mod.rs +++ b/src/reader/dialects/mod.rs @@ -117,6 +117,40 @@ pub(crate) fn inline_regex_flag( format!("CONCAT({}, {pattern})", dialect.sql_string_literal(flag)) } +/// A [`SqlDialect::sql_like`] pattern as a `GLOB` pattern: `%` and `_` +/// become `*` and `?`, a backslash-escaped character is literal, and `GLOB`'s +/// own `*`, `?` and `[` are bracketed so they match themselves. +pub(crate) fn like_to_glob(pattern: &str) -> String { + let mut out = String::with_capacity(pattern.len()); + let mut chars = pattern.chars(); + while let Some(ch) = chars.next() { + let literal = match ch { + '\\' => match chars.next() { + Some(next) => next, + None => break, + }, + '%' => { + out.push('*'); + continue; + } + '_' => { + out.push('?'); + continue; + } + other => other, + }; + match literal { + '*' | '?' | '[' => { + out.push('['); + out.push(literal); + out.push(']'); + } + other => out.push(other), + } + } + out +} + /// `1`/`0` boolean literal for backends without a real BOOLEAN literal /// (SQL Server, Oracle, SQLite). pub(crate) fn one_zero_boolean_literal(value: bool) -> String { @@ -232,7 +266,10 @@ mod tests { .sql_like("c", r"a\%", true) }; assert_eq!(like("duckdb"), r"c LIKE 'a\%' ESCAPE '\'"); - assert_eq!(like("mysql"), r"c LIKE 'a\\%' ESCAPE '\\'"); + assert_eq!( + like("mysql"), + r"CAST(c AS BINARY) LIKE CAST('a\\%' AS BINARY) ESCAPE '\\'" + ); assert_eq!(like("bigquery"), r"c LIKE 'a\\%'"); assert_eq!(like("clickhouse"), r"c LIKE 'a\\%'"); assert_eq!( @@ -243,6 +280,44 @@ mod tests { ); } + #[test] + fn case_sensitive_like_does_not_follow_a_case_folding_collation() { + let like = |scheme: &str| { + dialect_for_scheme(scheme) + .unwrap() + .sql_like("c", "a%", true) + }; + assert_eq!(like("sqlite"), "c GLOB 'a*'"); + assert_eq!( + like("mssql"), + r"c COLLATE Latin1_General_CS_AS LIKE 'a%' ESCAPE '\'" + ); + assert_eq!(like_to_glob(r"%a\%b\_*?[x]_"), "*a%b_[*][?][[]x]?"); + } + + #[cfg(feature = "sqlite")] + #[test] + fn case_sensitive_like_runs_in_sqlite() { + use crate::reader::Reader; + let reader = + crate::reader::SqliteReader::from_connection_string("sqlite://memory").unwrap(); + let p = SqliteDialect.sql_like("s", r"%A\_[%", true); + let df = reader + .execute_sql(&format!( + "SELECT s FROM (SELECT 'xA_[y' AS s UNION ALL SELECT 'xa_[y' UNION ALL SELECT 'xAb[y') WHERE {p}" + )) + .unwrap(); + assert_eq!(df.height(), 1); + } + + #[test] + fn an_empty_string_is_null_on_oracle() { + let oracle = dialect_for_scheme("oracle").unwrap(); + assert_eq!(oracle.sql_is_empty_string("c", false), "c IS NULL"); + assert_eq!(oracle.sql_is_empty_string("c", true), "c IS NOT NULL"); + assert_eq!(AnsiDialect.sql_is_empty_string("c", false), "c = ''"); + } + /// The default `LIKE` escape runs, with the backslash as written. #[cfg(feature = "duckdb")] #[test] @@ -279,14 +354,12 @@ mod tests { r"REGEXP_INSTR(c, '^a', 1, 1, 0, 'i') > 0" ); assert_eq!(regex("oracle", true).unwrap(), r"REGEXP_LIKE(c, '^a', 'c')"); - for scheme in [ - "clickhouse", - "trino", - "databricks", - "redshift", - "postgres", - "duckdb", - ] { + assert_eq!( + regex("duckdb", false).unwrap(), + "regexp_matches(c, '^a', 'i')" + ); + assert_eq!(regex("postgres", true).unwrap(), "c ~ '^a'"); + for scheme in ["clickhouse", "trino", "databricks", "redshift"] { assert!(regex(scheme, false).is_some(), "{scheme}"); } for scheme in ["sqlite", "mssql"] { @@ -447,26 +520,6 @@ mod tests { ); } - #[test] - fn regex_match_only_where_the_backend_has_one() { - let duckdb = dialect_for_scheme("duckdb").unwrap(); - assert_eq!( - duckdb.sql_regex_match("c", "'^a'", false).as_deref(), - Some("regexp_matches(c, '^a', 'i')") - ); - assert_eq!( - dialect_for_scheme("postgres") - .unwrap() - .sql_regex_match("c", "'^a'", true) - .as_deref(), - Some("c ~ '^a'") - ); - assert!(dialect_for_scheme("sqlite") - .unwrap() - .sql_regex_match("c", "'^a'", true) - .is_none()); - } - /// Contract for the quantile hook: callers pass the raw (unquoted) /// column name and the dialect quotes it. A name needing quoting must /// appear quoted — interpolating it raw breaks on any real column whose diff --git a/src/reader/dialects/mssql.rs b/src/reader/dialects/mssql.rs index 78188c610..dad517b47 100644 --- a/src/reader/dialects/mssql.rs +++ b/src/reader/dialects/mssql.rs @@ -24,9 +24,16 @@ impl SqlDialect for MssqlDialect { } // `[` opens a character class in T-SQL's `LIKE`, so it is escaped too. + // Comparison follows the column's collation, case-insensitive by + // default, so a case-sensitive match names a case-sensitive one. fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { let pattern = pattern.replace('[', "\\["); - super::like_with_escape_clause(self, expr, &pattern, case_sensitive) + if case_sensitive { + let expr = format!("{expr} COLLATE Latin1_General_CS_AS"); + super::like_with_escape_clause(self, &expr, &pattern, true) + } else { + super::like_with_escape_clause(self, expr, &pattern, false) + } } fn type_names(&self) -> crate::reader::TypeNames { diff --git a/src/reader/dialects/mysql.rs b/src/reader/dialects/mysql.rs index fdc4c7488..a19efd8da 100644 --- a/src/reader/dialects/mysql.rs +++ b/src/reader/dialects/mysql.rs @@ -10,6 +10,20 @@ use crate::reader::SqlDialect; pub struct MySqlDialect; impl SqlDialect for MySqlDialect { + // Comparison follows the column's collation, case-insensitive by + // default, so a case-sensitive match compares bytes. + fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { + if case_sensitive { + format!( + "CAST({expr} AS BINARY) LIKE CAST({} AS BINARY) ESCAPE {}", + self.sql_string_literal(pattern), + self.sql_string_literal("\\") + ) + } else { + super::like_with_escape_clause(self, expr, pattern, false) + } + } + // `REGEXP` is unanchored on MySQL 8 (ICU) and MariaDB (PCRE), and both read `(?i)` and `(?-i)`; its own case sensitivity follows the collation. fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { let pattern = super::inline_regex_flag(self, pattern, case_sensitive); diff --git a/src/reader/dialects/oracle.rs b/src/reader/dialects/oracle.rs index cffc68e95..6f6bcd6a3 100644 --- a/src/reader/dialects/oracle.rs +++ b/src/reader/dialects/oracle.rs @@ -15,6 +15,15 @@ use crate::reader::SqlDialect; pub struct OracleDialect; impl SqlDialect for OracleDialect { + // An empty string is NULL. + fn sql_is_empty_string(&self, expr: &str, negate: bool) -> String { + if negate { + format!("{expr} IS NOT NULL") + } else { + format!("{expr} IS NULL") + } + } + // Unanchored. fn sql_regex_match(&self, expr: &str, pattern: &str, case_sensitive: bool) -> Option { let flags = if case_sensitive { "c" } else { "i" }; diff --git a/src/reader/dialects/sqlite.rs b/src/reader/dialects/sqlite.rs index 596e4aa03..e0280f4aa 100644 --- a/src/reader/dialects/sqlite.rs +++ b/src/reader/dialects/sqlite.rs @@ -14,6 +14,18 @@ use crate::reader::SqlDialect; pub struct SqliteDialect; impl SqlDialect for SqliteDialect { + // `LIKE` ignores ASCII case, so a case-sensitive match is a `GLOB`. + fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { + if case_sensitive { + format!( + "{expr} GLOB {}", + self.sql_string_literal(&super::like_to_glob(pattern)) + ) + } else { + super::like_with_escape_clause(self, expr, pattern, false) + } + } + fn type_names(&self) -> crate::reader::TypeNames { crate::reader::TypeNames { number: Some("REAL"), diff --git a/src/reader/mod.rs b/src/reader/mod.rs index 43bc8f3ac..babd69830 100644 --- a/src/reader/mod.rs +++ b/src/reader/mod.rs @@ -479,6 +479,15 @@ pub trait SqlDialect { dialects::like_with_escape_clause(self, expr, pattern, case_sensitive) } + /// Whether `expr` is the empty string, or with `negate`, is not. + /// + /// Default compares with `''`. Override for backends that store the + /// empty string as NULL (Oracle). + fn sql_is_empty_string(&self, expr: &str, negate: bool) -> String { + let op = if negate { "<>" } else { "=" }; + format!("{expr} {op} {}", self.sql_string_literal("")) + } + /// One `ORDER BY` term sorting `expr` with its NULLs placed explicitly, /// since backends disagree on where they go by default. /// From 262dbf83cb0555a6545fa6913bb735ab3a53366b Mon Sep 17 00:00:00 2001 From: Sam Clark Date: Fri, 20 Nov 2026 14:50:52 -0600 Subject: [PATCH 8/8] Keep the Connections pane's typing for other backends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `kind_of` unwraps ClickHouse's `Nullable(…)` and `LowCardinality(…)` and knows ClickHouse, Oracle, DataFusion and SQLite names the shared exact match had dropped. - Oracle "is empty" stays `= ''`, as the view on DuckDB shows it; only "not empty" becomes `IS NOT NULL`. - `ggsql.resultsMemory` drops a byte count past `u64`, which would stop the kernel starting. - Tighten a cache test, document MySQL's byte-wise `_`, and note the time-zone timestamp display in the CHANGELOG. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++ ggsql-jupyter/AGENTS.md | 2 +- ggsql-jupyter/src/results/mod.rs | 42 ++++++++++++++++++++++----- ggsql-vscode/package.json | 2 +- ggsql-vscode/src/manager.ts | 14 +++++++-- ggsql-vscode/src/test/manager.test.ts | 4 ++- src/reader/cache.rs | 11 +------ src/reader/dialects/mod.rs | 4 +-- src/reader/dialects/mysql.rs | 4 ++- src/reader/dialects/oracle.rs | 6 ++-- 10 files changed, 65 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c613e71c..3713e87e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,9 @@ determines whether text boxes get drawn and additional settings `padding`, ### Fixed +- A timestamp with a time zone is now written as a timestamp in UTC + (`2024-01-02T03:04:05Z`) wherever ggsql shows a value as text, rather than + as the name of its Arrow type (#595). - `SCALE DISCRETE` now accepts integer columns, matching the documented ability to force numeric data into a discrete scale. Integer columns mapped to grouping aesthetics (e.g. `fill` in `DRAW bar`) can now be declared discrete diff --git a/ggsql-jupyter/AGENTS.md b/ggsql-jupyter/AGENTS.md index 778ca2acc..b1cfed498 100644 --- a/ggsql-jupyter/AGENTS.md +++ b/ggsql-jupyter/AGENTS.md @@ -198,7 +198,7 @@ So `RenderHints::canvas` picks a height (golden ratio, close to ggplot2's defaul - **Profiles are bounded too**: at most `MAX_FREQUENCY_VALUES` values in a frequency table, `MAX_QUANTILES` quantiles (each its own query) and `MAX_HISTOGRAM_BINS` bins. - **A page is bounded, and so is an export.** A page holds at most `MAX_PAGE_ROWS`, and an export at most `MAX_EXPORT_ROWS` (100,000), since it is built as text on the message loop, outside DuckDB's memory limit; copying a whole column of a longer result is an error that says so. For both, a range is cut short at the end of the view, and an index outside it is an error, so a cell never lands in another row's place. Scattered indices fetch just those rows. - **Filter values arrive as strings and are checked against the column's kind** before they reach the SQL: a number must parse as a finite number, a boolean as true/false, a date, datetime or time as ISO 8601 without an offset (it becomes the dialect's typed literal, `sql_date_literal` and friends, since not every backend compares a string with a temporal column), anything else becomes a quoted literal. A filter that cannot be built, or that the database rejects (tried on its own to find out which), is left out and its message returned in `errors`; the rest still apply. Filters are always ANDed. Identifiers, string and boolean literals, `LIKE` and regular expressions all come from the dialect the SQL is for (`SqlDialect::sql_string_literal`, `sql_like`, `sql_regex_match` — `None` where a backend has no regular expressions, as SQLite and SQL Server), so the same filter is written correctly for a backend that reads backslashes in literals, has no `ESCAPE` clause, or has more `LIKE` wildcards. Sort keys put NULLs last in either direction through `sql_order_term`, as the cache's DuckDB does, so exported SQL sorts as the view did. -- **Kinds are DuckDB type names matched exactly** (`kind_of`): a column type that only contains `INT`, like `POINT`, is not an integer, and an `ENUM` is a string. +- **Kinds are type names matched exactly** (`kind_of`): DuckDB's, and the names other backends report, which the Connections pane's `data_explorer.rs` shares. A column type that only contains `INT`, like `POINT`, is not an integer, an `ENUM` is a string, and ClickHouse's `Nullable(…)` and `LowCardinality(…)` are the type they wrap. - **`result_sql` writes SQL for the session's database**, not the cache: the cell's statements before the last as they were (the whole cell is one statement when it does not parse cleanly), then the last as a derived table with the view's filters and sort, in the session's dialect. A last statement that is not a query (`DESCRIBE`, `SHOW`, …) cannot be a derived table, so it is written as it was, with a comment saying the filters and sort were left out. It is in the kernel because only the kernel knows that dialect. A filter it cannot express there, or one on a column whose name the query repeats (renamed `a_1` in the cache, but ambiguous in SQL), is left out with a `--` comment. Measured at 10M rows (617 MB of Arrow data) on an Apple Silicon laptop: opening takes about 3.4 s, a sort 0.4–0.6 s, a page 20 ms, and memory levels off near 2 GB across repeated open/close cycles. Sorting the Arrow data directly avoids the copy but takes 2–5 s per stable sort, which is why the cache is DuckDB. diff --git a/ggsql-jupyter/src/results/mod.rs b/ggsql-jupyter/src/results/mod.rs index 19dd7b0a7..55dd558c8 100644 --- a/ggsql-jupyter/src/results/mod.rs +++ b/ggsql-jupyter/src/results/mod.rs @@ -463,9 +463,19 @@ fn gone() -> anyhow::Error { anyhow!("This query result is no longer available. Re-run the query to explore it.") } -/// A DuckDB type name's neutral kind. +/// A type name's neutral kind: DuckDB's names, and other backends' as the +/// Connections pane reports them. pub fn kind_of(type_name: &str) -> &'static str { - let upper = type_name.trim().to_uppercase(); + let mut upper = type_name.trim().to_uppercase(); + // ClickHouse wraps a type to say it may be NULL or is dictionary-encoded; + // what it holds is the type inside. + while let Some(inner) = ["NULLABLE(", "LOWCARDINALITY("].iter().find_map(|wrapper| { + upper + .strip_prefix(wrapper) + .and_then(|rest| rest.strip_suffix(')')) + }) { + upper = inner.trim().to_string(); + } // Nested types (`INTEGER[]`, `STRUCT(a INTEGER)`) name a scalar type // inside them, but are not one. if upper.ends_with(']') { @@ -482,11 +492,16 @@ pub fn kind_of(type_name: &str) -> &'static str { match base { "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INTEGER" | "BIGINT" | "HUGEINT" | "UTINYINT" | "USMALLINT" | "UINTEGER" | "UBIGINT" | "UHUGEINT" | "INT" | "INT1" | "INT2" | "INT4" - | "INT8" | "INT64" | "SERIAL" | "BIGSERIAL" | "SMALLSERIAL" => "integer", - "FLOAT" | "FLOAT4" | "FLOAT8" | "FLOAT64" | "REAL" | "DOUBLE" | "DOUBLE PRECISION" - | "DECIMAL" | "NUMERIC" | "BIGNUMERIC" | "NUMBER" => "floating", + | "INT8" | "INT16" | "INT32" | "INT64" | "INT128" | "INT256" | "UINT8" | "UINT16" + | "UINT32" | "UINT64" | "UINT128" | "UINT256" | "BIG INT" | "UNSIGNED BIG INT" + | "SERIAL" | "BIGSERIAL" | "SMALLSERIAL" => "integer", + "FLOAT" | "FLOAT4" | "FLOAT8" | "FLOAT16" | "FLOAT32" | "FLOAT64" | "REAL" | "DOUBLE" + | "DOUBLE PRECISION" | "BINARY_FLOAT" | "BINARY_DOUBLE" | "DECIMAL" | "DECIMAL32" + | "DECIMAL64" | "DECIMAL128" | "DECIMAL256" | "NUMERIC" | "BIGNUMERIC" | "NUMBER" => { + "floating" + } "BOOLEAN" | "BOOL" => "boolean", - "DATE" => "date", + "DATE" | "DATE32" | "DATE64" => "date", "TIMESTAMP" | "TIMESTAMP WITH TIME ZONE" | "TIMESTAMP WITHOUT TIME ZONE" @@ -498,13 +513,15 @@ pub fn kind_of(type_name: &str) -> &'static str { | "TIMESTAMP_LTZ" | "TIMESTAMP_TZ" | "DATETIME" + | "DATETIME64" | "DATETIME2" | "SMALLDATETIME" | "DATETIMEOFFSET" => "datetime", "TIME" | "TIME WITH TIME ZONE" | "TIME WITHOUT TIME ZONE" | "TIMETZ" => "time", "VARCHAR" | "VARCHAR2" | "NVARCHAR" | "NVARCHAR2" | "CHAR" | "NCHAR" | "BPCHAR" | "CHARACTER" | "CHARACTER VARYING" | "TEXT" | "TINYTEXT" | "MEDIUMTEXT" | "LONGTEXT" - | "NTEXT" | "CLOB" | "NCLOB" | "STRING" | "UUID" | "ENUM" => "string", + | "NTEXT" | "CLOB" | "NCLOB" | "STRING" | "FIXEDSTRING" | "UTF8" | "LARGEUTF8" | "UUID" + | "ENUM" | "ENUM8" | "ENUM16" => "string", _ => "unknown", } } @@ -1293,5 +1310,16 @@ mod tests { assert_eq!(kind_of("TIMESTAMP(6) WITH TIME ZONE"), "datetime"); assert_eq!(kind_of("character varying(255)"), "string"); assert_eq!(kind_of("DATETIME2"), "datetime"); + // ClickHouse, wrapped or not. + assert_eq!(kind_of("Nullable(Int32)"), "integer"); + assert_eq!(kind_of("UInt64"), "integer"); + assert_eq!(kind_of("Float32"), "floating"); + assert_eq!(kind_of("Nullable(DateTime64(3))"), "datetime"); + assert_eq!(kind_of("LowCardinality(Nullable(String))"), "string"); + // Oracle, DataFusion and SQLite. + assert_eq!(kind_of("BINARY_DOUBLE"), "floating"); + assert_eq!(kind_of("Date32"), "date"); + assert_eq!(kind_of("Utf8"), "string"); + assert_eq!(kind_of("UNSIGNED BIG INT"), "integer"); } } diff --git a/ggsql-vscode/package.json b/ggsql-vscode/package.json index 6c4fd5cc3..e15d4476e 100644 --- a/ggsql-vscode/package.json +++ b/ggsql-vscode/package.json @@ -175,7 +175,7 @@ "ggsql.resultsMemory": { "type": "string", "default": "1gb", - "pattern": "^\\s*[0-9]{1,12}\\s*([kKmMgG][bB])?\\s*$", + "pattern": "^\\s*[0-9]+\\s*([kKmMgG][bB])?\\s*$", "markdownDescription": "How much memory a ggsql console may use to keep query results for the Data Explorer, such as `512mb` or `2gb`. Results open in a Data Explorer are always kept; older ones make way for new ones. Takes effect in consoles started after the change." }, "ggsql.enableSqlFiles": { diff --git a/ggsql-vscode/src/manager.ts b/ggsql-vscode/src/manager.ts index 2102c37ac..937daa56b 100644 --- a/ggsql-vscode/src/manager.ts +++ b/ggsql-vscode/src/manager.ts @@ -549,6 +549,9 @@ export function consoleRowsEnv(rows: unknown): Record { : {}; } +/** The largest `u64`, the most bytes the kernel accepts for a budget. */ +const MAX_U64 = (1n << 64n) - 1n; + /** The most rows the kernel accepts for `--console-rows`, a `u32`. */ export const MAX_CONSOLE_ROWS = 4294967295; @@ -561,8 +564,15 @@ export function resultsMemoryEnv(value: unknown): Record { if (typeof value !== 'string') { return {}; } - const match = /^\s*([0-9]{1,12})\s*([kmg]b)?\s*$/i.exec(value); - return match && Number(match[1]) > 0 ? { GGSQL_RESULTS_MAX_BYTES: value.trim() } : {}; + const match = /^\s*([0-9]+)\s*([kmg]b)?\s*$/i.exec(value); + if (!match) { + return {}; + } + const unit = { kb: 1n << 10n, mb: 1n << 20n, gb: 1n << 30n }[match[2]?.toLowerCase() ?? ''] ?? 1n; + // The kernel reads the count as a `u64`, and refuses to start on zero or + // on more than that holds. + const bytes = BigInt(match[1]) * unit; + return bytes > 0n && bytes <= MAX_U64 ? { GGSQL_RESULTS_MAX_BYTES: value.trim() } : {}; } /** diff --git a/ggsql-vscode/src/test/manager.test.ts b/ggsql-vscode/src/test/manager.test.ts index 078f7bb11..7596f47e3 100644 --- a/ggsql-vscode/src/test/manager.test.ts +++ b/ggsql-vscode/src/test/manager.test.ts @@ -17,10 +17,12 @@ suite('manager', () => { assert.deepStrictEqual(resultsMemoryEnv(' 512MB '), { GGSQL_RESULTS_MAX_BYTES: '512MB' }); assert.deepStrictEqual(resultsMemoryEnv('2gb'), { GGSQL_RESULTS_MAX_BYTES: '2gb' }); assert.deepStrictEqual(resultsMemoryEnv('1048576'), { GGSQL_RESULTS_MAX_BYTES: '1048576' }); + assert.deepStrictEqual(resultsMemoryEnv('18446744073709551615'), { GGSQL_RESULTS_MAX_BYTES: '18446744073709551615' }); }); test('resultsMemoryEnv leaves out a value the kernel would refuse', () => { - for (const value of ['', '0', '0gb', '1.5gb', '2 terabytes', '-1mb', '1234567890123gb', 512, undefined]) { + // 99999999999gb and 18446744073709551616 overflow a u64. + for (const value of ['', '0', '0gb', '1.5gb', '2 terabytes', '-1mb', '99999999999gb', '18446744073709551616', 512, undefined]) { assert.deepStrictEqual(resultsMemoryEnv(value), {}, String(value)); } }); diff --git a/src/reader/cache.rs b/src/reader/cache.rs index 6230ea93b..fa1058a91 100644 --- a/src/reader/cache.rs +++ b/src/reader/cache.rs @@ -1531,16 +1531,7 @@ mod behavior_tests { fn test_only_an_entry_can_be_pinned() { let tables = tiny_budget_tables(); assert!(tables.set_pinned("nothing", true).is_err()); - // No pin was left behind to keep a later entry of that key. - tables - .put( - "nothing", - "q", - "__t_n__", - df! { "x" => vec![1_i64] }.unwrap(), - ) - .unwrap(); - assert!(!tables.contains("nothing").unwrap()); + assert_eq!(tables.pinned_bytes().unwrap(), 0); } #[test] diff --git a/src/reader/dialects/mod.rs b/src/reader/dialects/mod.rs index 057df7584..02e063676 100644 --- a/src/reader/dialects/mod.rs +++ b/src/reader/dialects/mod.rs @@ -311,9 +311,9 @@ mod tests { } #[test] - fn an_empty_string_is_null_on_oracle() { + fn not_empty_is_not_null_on_oracle() { let oracle = dialect_for_scheme("oracle").unwrap(); - assert_eq!(oracle.sql_is_empty_string("c", false), "c IS NULL"); + assert_eq!(oracle.sql_is_empty_string("c", false), "c = ''"); assert_eq!(oracle.sql_is_empty_string("c", true), "c IS NOT NULL"); assert_eq!(AnsiDialect.sql_is_empty_string("c", false), "c = ''"); } diff --git a/src/reader/dialects/mysql.rs b/src/reader/dialects/mysql.rs index a19efd8da..b40cbd7b8 100644 --- a/src/reader/dialects/mysql.rs +++ b/src/reader/dialects/mysql.rs @@ -11,7 +11,9 @@ pub struct MySqlDialect; impl SqlDialect for MySqlDialect { // Comparison follows the column's collation, case-insensitive by - // default, so a case-sensitive match compares bytes. + // default, so a case-sensitive match compares bytes. `_` then matches one + // byte rather than one character, which an escaped search term never + // relies on. fn sql_like(&self, expr: &str, pattern: &str, case_sensitive: bool) -> String { if case_sensitive { format!( diff --git a/src/reader/dialects/oracle.rs b/src/reader/dialects/oracle.rs index 6f6bcd6a3..4c0a0989f 100644 --- a/src/reader/dialects/oracle.rs +++ b/src/reader/dialects/oracle.rs @@ -15,12 +15,14 @@ use crate::reader::SqlDialect; pub struct OracleDialect; impl SqlDialect for OracleDialect { - // An empty string is NULL. + // An empty string is NULL, so `<> ''` is never true and "not empty" is + // `IS NOT NULL`. "Empty" stays `= ''`, which matches nothing here, as no + // value read from Oracle is the empty string. fn sql_is_empty_string(&self, expr: &str, negate: bool) -> String { if negate { format!("{expr} IS NOT NULL") } else { - format!("{expr} IS NULL") + format!("{expr} = ''") } }
{}