diff --git a/.gitignore b/.gitignore
index a8dd00ef04..1a11791125 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,7 @@ __pycache__/
.clangd
.DS_Store
.emsdk
+.binaryen
.plan
.pnpm-store
.ipynb_checkpoints
@@ -71,3 +72,5 @@ rust/perspective-server/build
target/
dist-gh-pages
rust/perspective-python/perspective/widget/static
+docs/static/projects
+docs/static/data
diff --git a/Cargo.lock b/Cargo.lock
index 2278da4d64..aeb3c694a8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -430,8 +430,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a"
dependencies = [
"find-msvc-tools",
- "jobserver",
- "libc",
"shlex",
]
@@ -482,7 +480,7 @@ version = "4.5.49"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"syn 2.0.111",
@@ -510,18 +508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
dependencies = [
"termcolor",
- "unicode-width 0.1.14",
-]
-
-[[package]]
-name = "codespan-reporting"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
-dependencies = [
- "serde",
- "termcolor",
- "unicode-width 0.2.2",
+ "unicode-width",
]
[[package]]
@@ -677,68 +664,6 @@ dependencies = [
"syn 2.0.111",
]
-[[package]]
-name = "cxx"
-version = "1.0.190"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7620f6cfc4dcca21f2b085b7a890e16c60fd66f560cd69ee60594908dc72ab1"
-dependencies = [
- "cc",
- "cxx-build",
- "cxxbridge-cmd",
- "cxxbridge-flags",
- "cxxbridge-macro",
- "foldhash 0.2.0",
- "link-cplusplus",
-]
-
-[[package]]
-name = "cxx-build"
-version = "1.0.190"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a9bc1a22964ff6a355fbec24cf68266a0ed28f8b84c0864c386474ea3d0e479"
-dependencies = [
- "cc",
- "codespan-reporting 0.13.1",
- "indexmap 2.14.0",
- "proc-macro2",
- "quote",
- "scratch",
- "syn 2.0.111",
-]
-
-[[package]]
-name = "cxxbridge-cmd"
-version = "1.0.190"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1f29a879d35f7906e3c9b77d7a1005a6a0787d330c09dfe4ffb5f617728cb44"
-dependencies = [
- "clap",
- "codespan-reporting 0.13.1",
- "indexmap 2.14.0",
- "proc-macro2",
- "quote",
- "syn 2.0.111",
-]
-
-[[package]]
-name = "cxxbridge-flags"
-version = "1.0.190"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d67109015f93f683e364085aa6489a5b2118b4a40058482101d699936a7836d6"
-
-[[package]]
-name = "cxxbridge-macro"
-version = "1.0.190"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d187e019e7b05a1f3e69a8396b70800ee867aa9fc2ab972761173ccee03742df"
-dependencies = [
- "indexmap 2.14.0",
- "proc-macro2",
- "quote",
- "syn 2.0.111",
-]
-
[[package]]
name = "dashmap"
version = "5.5.3"
@@ -947,12 +872,6 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
-[[package]]
-name = "foldhash"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
-
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -1344,7 +1263,7 @@ version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
- "foldhash 0.1.5",
+ "foldhash",
"serde",
]
@@ -1360,12 +1279,6 @@ version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
[[package]]
name = "heck"
version = "0.5.0"
@@ -1633,16 +1546,6 @@ dependencies = [
"syn 2.0.111",
]
-[[package]]
-name = "jobserver"
-version = "0.1.34"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
-dependencies = [
- "getrandom 0.3.4",
- "libc",
-]
-
[[package]]
name = "js-sys"
version = "0.3.85"
@@ -2090,7 +1993,6 @@ version = "0.0.0"
dependencies = [
"clap",
"wasm-bindgen-cli-support",
- "wasm-opt",
]
[[package]]
@@ -2456,7 +2358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
dependencies = [
"bytes",
- "heck 0.5.0",
+ "heck",
"itertools 0.12.1",
"log",
"multimap",
@@ -2601,7 +2503,7 @@ version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"pyo3-build-config",
"quote",
@@ -2873,12 +2775,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-[[package]]
-name = "scratch"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2"
-
[[package]]
name = "seahash"
version = "4.1.0"
@@ -3148,12 +3044,6 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-[[package]]
-name = "strum"
-version = "0.24.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f"
-
[[package]]
name = "strum"
version = "0.26.3"
@@ -3172,26 +3062,13 @@ dependencies = [
"strum_macros 0.27.2",
]
-[[package]]
-name = "strum_macros"
-version = "0.24.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59"
-dependencies = [
- "heck 0.4.1",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 1.0.109",
-]
-
[[package]]
name = "strum_macros"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"rustversion",
@@ -3204,7 +3081,7 @@ version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"syn 2.0.111",
@@ -3677,12 +3554,6 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
-[[package]]
-name = "unicode-width"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
-
[[package]]
name = "unindent"
version = "0.2.4"
@@ -3768,7 +3639,7 @@ version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef06db404cbaed87cb25fd2ca3a62502af485f43383c9641ffcf1479d02fffd"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"syn 2.0.111",
@@ -3937,46 +3808,6 @@ dependencies = [
"wasmparser",
]
-[[package]]
-name = "wasm-opt"
-version = "0.116.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fd87a4c135535ffed86123b6fb0f0a5a0bc89e50416c942c5f0662c645f679c"
-dependencies = [
- "anyhow",
- "libc",
- "strum 0.24.1",
- "strum_macros 0.24.3",
- "tempfile",
- "thiserror 1.0.69",
- "wasm-opt-cxx-sys",
- "wasm-opt-sys",
-]
-
-[[package]]
-name = "wasm-opt-cxx-sys"
-version = "0.116.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c57b28207aa724318fcec6575fe74803c23f6f266fce10cbc9f3f116762f12e"
-dependencies = [
- "anyhow",
- "cxx",
- "cxx-build",
- "wasm-opt-sys",
-]
-
-[[package]]
-name = "wasm-opt-sys"
-version = "0.116.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a1cce564dc768dacbdb718fc29df2dba80bd21cb47d8f77ae7e3d95ceb98cbe"
-dependencies = [
- "anyhow",
- "cc",
- "cxx",
- "cxx-build",
-]
-
[[package]]
name = "wasmparser"
version = "0.240.0"
@@ -4236,7 +4067,7 @@ dependencies = [
"basic-toml",
"bumpalo",
"clap",
- "codespan-reporting 0.11.1",
+ "codespan-reporting",
"diffy",
"dirs",
"either",
diff --git a/README.md b/README.md
index 6df4065c61..83692593ac 100644
--- a/README.md
+++ b/README.md
@@ -13,67 +13,103 @@
[](https://pypi.python.org/pypi/perspective-python)
[](https://crates.io/crates/perspective)
-
-
Perspective is an interactive analytics and data visualization component for
-large and streaming datasets. Build user-configurable reports, dashboards,
-notebooks, and applications with a high-performance query engine compiled to
-WebAssembly, Python, and Rust.
+large, real-time and streaming datasets. Build user-configurable reports,
+dashboards, notebooks and applications, backed by a high-performance streaming
+query engine that runs in-browser via WebAssembly or server-side in Python,
+Node.js and Rust — or delegates to a database you already have.
+
+
+
+
+ 
+
## Features
-- A framework-agnostic user interface packaged as a
+- A data-reactive UI packaged as a
[Custom Element](https://developer.mozilla.org/en-US/docs/Web/Web_Components/Using_custom_elements),
- which connects to a Data Model in-browser (via WebAssembly) or remotely (via
- WebSocket, with integration in Python, Node.js and Rust). Includes a data
- grid, 10+ chart types line, bar, area, scatter, heatmap, treemap, sunburst,
- candlestick, and more.
-
-- A Data Model API for pluggable engines, enabling Perspective's UI to query
- external data sources like [DuckDB](https://duckdb.org/) while translating
- view configurations into native queries.
-
-- A fast, memory-efficient streaming Data Model built-in, written in C++ and
- compiled for [WebAssembly](https://webassembly.org/),
- [Python](https://www.python.org/), and [Rust](https://www.rust-lang.org/).
- Supports read/write/streaming for [Apache Arrow](https://arrow.apache.org/),
- with a columnar expression language based on
- [ExprTK](https://github.com/ArashPartow/exprtk).
-
-- A [JupyterLab](https://jupyter.org/) widget and Python client library for
- interactive data analysis in notebooks.
+ with drag-and-drop query and layout configuration. Includes a virtual-scrolling,
+ editable data grid, WebGL charting engine with 15+ chart types, tile-based
+ geographic maps, full theme support, and [React](https://react.dev/) bindings.
+
+- A fast, memory-efficient streaming query engine written in C++ and compiled
+ for [WebAssembly](https://webassembly.org/) (including a 64-bit `memory64`
+ build for in-browser datasets larger than 4GB),
+ [Python](https://www.python.org/) and [Rust](https://www.rust-lang.org/).
+ Tables update incrementally and views tick in real time, with reactive
+ joins across tables, a columnar expression language based on
+ [ExprTK](https://github.com/ArashPartow/exprtk), and read/write/streaming
+ support for [Apache Arrow](https://arrow.apache.org/), CSV and JSON.
+
+- A symmetric client/server architecture — the same Client API connects to an
+ engine in-process, in a Web Worker, or remotely over WebSocket, with server
+ bindings for Python (aiohttp, Starlette, Tornado), Node.js and Rust.
+ Datasets can be mirrored to the browser for fluid interaction or virtualized
+ server-side, streaming only what's visible.
+
+- Virtual servers that run Perspective's UI directly on external engines like
+ [DuckDB](https://duckdb.org/), [ClickHouse](https://clickhouse.com/) and
+ [Polars](https://pola.rs/), translating view configurations into native
+ queries — no ETL or data copy required.
+
+- A [Jupyter](https://jupyter.org/) widget built on
+ [anywidget](https://anywidget.dev/) and a Python client library for
+ interactive data analysis in JupyterLab and other notebook environments.
## Documentation
- [Project Site](https://perspective-dev.github.io/)
- [User Guide](https://perspective-dev.github.io/guide/)
- JavaScript API
- - [`@perspective-dev/client` Browser](https://perspective-dev.github.io/browser/modules/src_ts_perspective.browser.ts.html)
- - [`@perspective-dev/client` Node.js](https://perspective-dev.github.io/node/modules/src_ts_perspective.node.ts.html)
+ - [`@perspective-dev/react` React Component](https://perspective-dev.github.io/react/index.html)
+ - [`@perspective-dev/viewer` Web Component](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html)
+ - [`@perspective-dev/client` Client (Browser)](https://perspective-dev.github.io/browser/modules/src_ts_perspective.browser.ts.html)
+ - [`@perspective-dev/client` Client (Node.js)](https://perspective-dev.github.io/node/modules/src_ts_perspective.node.ts.html)
- [`@perspective-dev/client` Clickhouse Virtual Server](https://perspective-dev.github.io/browser/modules/dist_esm_virtual_servers_clickhouse.js.html)
- [`@perspective-dev/client` DuckDB Virtual Server](https://perspective-dev.github.io/browser/modules/dist_esm_virtual_servers_duckdb.js.html)
- - [`@perspective-dev/viewer` Web Component](https://perspective-dev.github.io/viewer/modules/perspective-viewer.html)
- Python API
- [`perspective`](https://perspective-dev.github.io/python/index.html)
- [`perspective.widget`](https://perspective-dev.github.io/python/perspective/widget.html)
- [`perspective.handlers.aiohttp`](https://perspective-dev.github.io/python/perspective/handlers/aiohttp.html)
- - [`perspective.handlers.starlette`](https://perspective-dev.github.io/python/perspective/handlers/starlett.html)
+ - [`perspective.handlers.starlette`](https://perspective-dev.github.io/python/perspective/handlers/starlette.html)
- [`perspective.handlers.tornado`](https://perspective-dev.github.io/python/perspective/handlers/tornado.html)
- [`perspective.virtual_servers.clickhouse`](https://perspective-dev.github.io/python/perspective/virtual_servers/clickhouse.html)
- [`perspective.virtual_servers.duckdb`](https://perspective-dev.github.io/python/perspective/virtual_servers/duckdb.html)
- Rust API
- [`perspective`](https://docs.rs/perspective/latest/perspective/)
- - [`perspective-client`](https://docs.rs/perspective-client/latest/perspective_client/)
- - [`perspective-server`](https://docs.rs/perspective-server/latest/perspective_server/)
- - [`perspective-python`](https://docs.rs/perspective-python/latest/perspective_python/)
- - [`perspective-js`](https://docs.rs/perspective-js/latest/perspective_js/)
- - [`perspective-viewer`](https://docs.rs/perspective-viewer/latest/perspective_viewer/)
+
## Examples
-
-| editable | file | duckdb |
 |  |  |
| fractal | market | raycasting |
 |  |  |
| evictions | nypd | streaming |
 |  |  |
| covid | webcam | movies |
 |  |  |
| superstore | olympics | dataset |
 |  |  |
-
+
+
+
+| Superstore | Workspace | Webcam |
+
+
+ |
+ |
+ |
+
+
+| Raycasting | Market | NYPD |
+
+
+ |
+ |
+ |
+
+
+| Movies | Evictions | Fractal |
+
+
+ |
+ |
+ |
+
+
+
## Media
@@ -102,6 +138,7 @@ WebAssembly, Python, and Rust.
---
+
diff --git a/docs/build.collage.mjs b/docs/build.collage.mjs
new file mode 100644
index 0000000000..a1125e216a
--- /dev/null
+++ b/docs/build.collage.mjs
@@ -0,0 +1,166 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
+// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors. ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+export const COLLAGE = "collage.png";
+
+/** Ceiling on waiting for the collage's tiles to finish loading. */
+const SETTLE_TIMEOUT = 60_000;
+
+const COLLAGE_WIDTH = 1600;
+const COLLAGE_ASPECT = 16 / 8;
+const COLLAGE_GAP = 2;
+
+const COLLAGE_BG = { light: "#ffffff", dark: "#242526" };
+
+const COLLAGE_SEED = 0x5eed;
+
+function shuffled(items, seed) {
+ let state = seed;
+ const random = () => {
+ state = (state + 0x6d2b79f5) | 0;
+ let t = Math.imul(state ^ (state >>> 15), 1 | state);
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+ };
+
+ const out = items.slice();
+ for (let i = out.length - 1; i > 0; i--) {
+ const j = Math.floor(random() * (i + 1));
+ [out[i], out[j]] = [out[j], out[i]];
+ }
+
+ return out;
+}
+
+function pngSize(file) {
+ const fd = fs.openSync(file, "r");
+ try {
+ const head = Buffer.alloc(24);
+ fs.readSync(fd, head, 0, 24, 0);
+ return { width: head.readUInt32BE(16), height: head.readUInt32BE(20) };
+ } finally {
+ fs.closeSync(fd);
+ }
+}
+
+function bestFitGrid(count, aspect, width, height) {
+ let best;
+ for (let cols = 1; cols <= count; cols++) {
+ const rows = Math.ceil(count / cols);
+ const tileWidth = Math.floor((width - (cols - 1) * COLLAGE_GAP) / cols);
+
+ const tileHeight = Math.round(tileWidth / aspect);
+ const contentHeight = rows * tileHeight + (rows - 1) * COLLAGE_GAP;
+ if (contentHeight <= height && tileHeight > (best?.tileHeight ?? 0)) {
+ best = { cols, rows, tileWidth, tileHeight };
+ }
+ }
+
+ return (
+ best ?? {
+ cols: count,
+ rows: 1,
+ tileWidth: Math.floor(height * aspect),
+ tileHeight: height,
+ }
+ );
+}
+
+export async function collage(page, ids, theme, { out, port }) {
+ const dir = path.join(out, theme);
+ const present = shuffled(
+ ids.filter((id) => fs.existsSync(path.join(dir, `${id}.png`))),
+ COLLAGE_SEED,
+ );
+
+ if (present.length === 0) {
+ console.warn(` ✗ ${theme} collage: no thumbnails to composite.`);
+ return;
+ }
+
+ const height = Math.round(COLLAGE_WIDTH / COLLAGE_ASPECT);
+ const sample = pngSize(path.join(dir, `${present[0]}.png`));
+ const { cols, rows, tileWidth, tileHeight } = bestFitGrid(
+ present.length,
+ sample.width / sample.height,
+ COLLAGE_WIDTH,
+ height,
+ );
+
+ const tiles = present
+ .map(
+ (id) =>
+ `
`,
+ )
+ .join("");
+
+ await page.setViewport({ width: COLLAGE_WIDTH, height });
+ await page.setContent(
+ `
+ ${tiles}
`,
+ { waitUntil: "load" },
+ );
+
+ const tiles_found = await page.$$eval(".collage img", (x) => x.length);
+ if (tiles_found !== present.length) {
+ throw new Error(
+ `collage page has ${tiles_found} tiles, expected ${present.length}`,
+ );
+ }
+
+ await page.waitForFunction(
+ () =>
+ [...document.querySelectorAll(".collage img")].every(
+ (x) => x.complete,
+ ),
+ { timeout: SETTLE_TIMEOUT },
+ );
+
+ const broken = await page.$$eval(
+ ".collage img",
+ (images) => images.filter((x) => x.naturalWidth === 0).length,
+ );
+
+ if (broken > 0) {
+ throw new Error(`${broken} thumbnail(s) failed to load`);
+ }
+
+ fs.writeFileSync(path.join(dir, COLLAGE), await page.screenshot());
+ console.log(
+ `Collage (${theme}): ${present.length} thumbnails, ${cols}×${rows} ` +
+ `grid of ${tileWidth}×${tileHeight} tiles, ` +
+ `${COLLAGE_WIDTH}×${height}.`,
+ );
+}
diff --git a/docs/build.config.mjs b/docs/build.config.mjs
index 0dec5d772a..555d2d3abf 100644
--- a/docs/build.config.mjs
+++ b/docs/build.config.mjs
@@ -12,6 +12,7 @@
import * as esbuild from "esbuild";
import * as fs from "node:fs";
+import * as http from "node:http";
import * as path from "node:path";
import { createRequire } from "module";
import { bundleAsync as bundleCssAsync, composeVisitors } from "lightningcss";
@@ -20,6 +21,14 @@ import { fileURLToPath } from "node:url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DIST = path.join(__dirname, "dist");
+const WATCH = process.argv.includes("--watch");
+const RELOAD_PORT = Number(
+ process.argv.find((x) => x.startsWith("--reload-port="))?.split("=")[1] ??
+ 8081,
+);
+
+const HTML_PAGES = ["index.html"];
+
function copyRecursive(src, dest) {
if (!fs.existsSync(src)) return;
const stat = fs.statSync(src);
@@ -33,7 +42,11 @@ function copyRecursive(src, dest) {
}
}
-// Inline url() asset references as data URIs.
+/**
+ * A lightningcss visitor inlining `url()` asset references as data URIs.
+ *
+ * @param fromFile the stylesheet relative paths resolve against.
+ */
export function inlineUrlVisitor(fromFile) {
const dir = path.dirname(fromFile);
return composeVisitors([
@@ -47,7 +60,6 @@ export function inlineUrlVisitor(fromFile) {
const resolved = path.resolve(dir, url.url);
if (!fs.existsSync(resolved)) {
throw new Error(`File not found ${url.url}`);
- // return;
}
const content = fs.readFileSync(resolved);
@@ -73,6 +85,12 @@ export function inlineUrlVisitor(fromFile) {
]);
}
+/**
+ * A lightningcss resolver reading `node_modules` specifiers and leaving
+ * `http` imports external.
+ *
+ * @param url the module URL bare specifiers resolve against.
+ */
export const resolveNPM = (url) => ({
read(filePath) {
if (filePath.startsWith("http")) {
@@ -96,33 +114,134 @@ export const resolveNPM = (url) => ({
},
});
-async function build() {
- // Clean and create dist
- fs.mkdirSync(DIST, { recursive: true });
+const RELOAD_CLIENTS = new Set();
+
+function startReloadServer() {
+ const server = http.createServer((request, response) => {
+ if (!request.url.startsWith("/livereload")) {
+ response.writeHead(404).end();
+ return;
+ }
+
+ response.writeHead(200, {
+ "Content-Type": "text/event-stream",
+ "Cache-Control": "no-cache",
+ Connection: "keep-alive",
+ "Access-Control-Allow-Origin": "*",
+ });
+
+ response.write("retry: 500\n\n");
+ RELOAD_CLIENTS.add(response);
+ request.on("close", () => RELOAD_CLIENTS.delete(response));
+ });
+
+ server.on("error", (e) => {
+ if (e.code === "EADDRINUSE") {
+ console.error(
+ `Live-reload port ${RELOAD_PORT} is in use. Pass ` +
+ `--reload-port= (and restart the browser tab).`,
+ );
+ } else {
+ console.error(e);
+ }
+ });
+
+ server.listen(RELOAD_PORT);
+ return server;
+}
+
+function notifyReload(reason) {
+ console.log(` ↻ ${reason}`);
+ for (const client of RELOAD_CLIENTS) {
+ client.write(`event: reload\ndata: ${reason}\n\n`);
+ }
+}
+
+const RELOAD_SNIPPET = `
+`;
- // Bundle CSS
- const { code: cssCode } = await bundleCssAsync({
+async function buildCss() {
+ const { code } = await bundleCssAsync({
filename: path.join(__dirname, "./src/css/style.css"),
- minify: true,
+ minify: !WATCH,
resolver: resolveNPM(import.meta.url),
visitor: inlineUrlVisitor("./src/css/style.css"),
});
fs.mkdirSync(path.join(DIST, "css"), { recursive: true });
- fs.writeFileSync(path.join(DIST, "style.css"), cssCode);
-
- // Bundle JS entry points
- await esbuild.build({
- entryPoints: [
- path.join(__dirname, "src/index.ts"),
- path.join(__dirname, "src/examples.ts"),
- path.join(__dirname, "src/block.ts"),
- ],
+ fs.writeFileSync(path.join(DIST, "style.css"), code);
+}
+
+function copyHtml() {
+ for (const html of HTML_PAGES) {
+ const source = fs.readFileSync(
+ path.join(__dirname, "src", html),
+ "utf8",
+ );
+
+ const output = WATCH
+ ? source.replace(" ", `${RELOAD_SNIPPET} `)
+ : source;
+
+ fs.writeFileSync(path.join(DIST, html), output);
+ }
+}
+
+function copyStatic() {
+ copyRecursive(path.join(__dirname, "static"), DIST);
+ const arrow = path.join(
+ __dirname,
+ "node_modules/superstore-arrow/superstore.lz4.arrow",
+ );
+
+ fs.mkdirSync(path.join(DIST, "data"), { recursive: true });
+ if (fs.existsSync(arrow)) {
+ fs.copyFileSync(arrow, path.join(DIST, "data/superstore.lz4.arrow"));
+ } else {
+ console.warn("Missing superstore-arrow; Superstore Projects will 404.");
+ }
+}
+
+function copyDocsBundle() {
+ const docs_bundle = path.join(
+ __dirname,
+ "node_modules/@perspective-dev/viewer/dist/docs/perspective-docs.json",
+ );
+
+ if (fs.existsSync(docs_bundle)) {
+ fs.copyFileSync(docs_bundle, path.join(DIST, "perspective-docs.json"));
+ } else {
+ console.warn(
+ "No perspective-docs.json; the agent's `search_docs` tool will " +
+ "be unavailable until `@perspective-dev/viewer` is built.",
+ );
+ }
+}
+
+function esbuildOptions() {
+ return {
+ entryPoints: [path.join(__dirname, "src/index.ts")],
bundle: true,
splitting: true,
format: "esm",
outdir: DIST,
- minify: true,
+ minify: !WATCH,
sourcemap: true,
target: ["es2022"],
define: {
@@ -132,45 +251,118 @@ async function build() {
".wasm": "file",
".arrow": "file",
},
- });
+ };
+}
- // Copy HTML files
- for (const html of ["index.html", "examples.html", "block.html"]) {
- fs.copyFileSync(
- path.join(__dirname, "src", html),
- path.join(DIST, html),
- );
+async function build() {
+ fs.mkdirSync(DIST, { recursive: true });
+ await buildCss();
+ await esbuild.build(esbuildOptions());
+ copyHtml();
+ copyStatic();
+ copyDocsBundle();
+ console.log("Build complete: dist/");
+}
+
+function debounce(fn, ms = 60) {
+ let timer;
+ return () => {
+ clearTimeout(timer);
+ timer = setTimeout(fn, ms);
+ };
+}
+
+function watchDir(dir, handler) {
+ if (!fs.existsSync(dir)) {
+ return;
}
- // Copy static assets
- copyRecursive(path.join(__dirname, "static"), DIST);
+ fs.watch(dir, { recursive: true }, debounce(handler));
+}
- // Generate blocks manifest
- const blocksDir = path.join(DIST, "blocks");
- if (fs.existsSync(blocksDir)) {
- const manifest = {};
- for (const example of fs.readdirSync(blocksDir)) {
- const exDir = path.join(blocksDir, example);
- if (!fs.statSync(exDir).isDirectory()) continue;
- manifest[example] = fs
- .readdirSync(exDir)
- .filter(
- (f) =>
- !f.startsWith(".") &&
- !f.endsWith(".png") &&
- !f.endsWith(".arrow"),
- );
- }
- fs.writeFileSync(
- path.join(blocksDir, "manifest.json"),
- JSON.stringify(manifest),
- );
+async function guard(label, step) {
+ try {
+ await step();
+ notifyReload(label);
+ } catch (e) {
+ console.error(` ✗ ${label} failed:\n${e.message ?? e}`);
}
+}
- console.log("Build complete: dist/");
+async function watch() {
+ fs.mkdirSync(DIST, { recursive: true });
+ try {
+ await buildCss();
+ } catch (e) {
+ console.error(` ✗ css failed:\n${e.message ?? e}`);
+ }
+
+ copyHtml();
+ copyStatic();
+ copyDocsBundle();
+
+ const ctx = await esbuild.context({
+ ...esbuildOptions(),
+ plugins: [
+ {
+ name: "livereload",
+ setup(build) {
+ let first = true;
+ build.onEnd((result) => {
+ if (result.errors.length > 0) {
+ console.error(
+ ` ✗ js failed (${result.errors.length} error(s))`,
+ );
+ } else if (first) {
+ first = false;
+ } else {
+ notifyReload("js");
+ }
+ });
+ },
+ },
+ ],
+ });
+
+ await ctx.watch();
+ watchDir(path.join(__dirname, "src/css"), () => guard("css", buildCss));
+ watchDir(path.join(__dirname, "static"), () => guard("static", copyStatic));
+ fs.watch(
+ path.join(__dirname, "src"),
+ { recursive: true },
+ debounce(() => {
+ if (HTML_PAGES.some((p) => hasChanged(p))) {
+ guard("html", copyHtml);
+ }
+ }),
+ );
+
+ const server = startReloadServer();
+ console.log(
+ `Watching docs/src — live reload on :${RELOAD_PORT}.\n` +
+ `Serve dist/ separately (\`pnpm start\`); Ctrl-C to stop.`,
+ );
+
+ for (const signal of ["SIGINT", "SIGTERM"]) {
+ process.on(signal, () => {
+ ctx.dispose();
+ server.close();
+ process.exit(0);
+ });
+ }
+}
+
+function hasChanged(page) {
+ const from = path.join(__dirname, "src", page);
+ const to = path.join(DIST, page);
+ if (!fs.existsSync(to)) {
+ return true;
+ }
+
+ return fs.statSync(from).mtimeMs > fs.statSync(to).mtimeMs;
}
-build().catch((e) => {
+(WATCH ? watch() : build()).catch((e) => {
console.error(e);
process.exit(1);
});
diff --git a/docs/build.mjs b/docs/build.mjs
deleted file mode 100644
index 1df02a3957..0000000000
--- a/docs/build.mjs
+++ /dev/null
@@ -1,157 +0,0 @@
-// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
-// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
-// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
-// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
-// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
-// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
-// ┃ Copyright (c) 2017, the Perspective Authors. ┃
-// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
-// ┃ This file is part of the Perspective library, distributed under the terms ┃
-// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
-// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
-
-import puppeteer from "puppeteer";
-import * as fs from "node:fs";
-import * as cp from "node:child_process";
-import * as path from "node:path";
-import { fileURLToPath } from "node:url";
-
-import EXAMPLES from "./src/data/features.js";
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-// features.js uses CJS exports.default, import it dynamically
-// const EXAMPLES = (await import("./src/data/features.ts")).default;
-
-const perspective = import(
- "@perspective-dev/client/dist/esm/perspective.node.js"
-);
-
-const DEFAULT_VIEWPORT = {
- width: 400,
- height: 300,
-};
-
-async function run_with_theme(page, is_dark = false, order) {
- await page.goto("http://localhost:8080/");
- await page.setContent(template(is_dark));
- await page.setViewport(DEFAULT_VIEWPORT);
- await page.evaluate(async () => {
- while (!window.__TEST_PERSPECTIVE_READY__) {
- await new Promise((resolve) => setTimeout(resolve, 10));
- }
- });
-
- await page.evaluate(async function () {
- const viewer = document.querySelector("perspective-viewer");
- await viewer.flush();
- await viewer.toggleConfig();
- });
-
- for (const idx in EXAMPLES) {
- const { config, viewport } = EXAMPLES[idx];
- await page.setViewport(viewport || DEFAULT_VIEWPORT);
- const new_config = Object.assign(
- {
- plugin: "Datagrid",
- group_by: [],
- expressions: {},
- split_by: [],
- sort: [],
- aggregates: {},
- },
- config,
- );
- console.log(JSON.stringify(new_config));
-
- await page.evaluate(async (config) => {
- const viewer = document.querySelector("perspective-viewer");
- await viewer.reset();
- await viewer.restore(config);
- }, new_config);
-
- const screenshot = await page.screenshot({
- captureBeyondViewport: false,
- fullPage: true,
- });
-
- const name = `static/features/feature_${idx}${
- is_dark ? "_dark" : ""
- }.png`;
-
- fs.writeFileSync(name, screenshot);
- cp.execSync(`convert ${name} -resize 200x150 ${name}`);
- }
-
- const suffix = is_dark ? "_dark" : "";
- const montage_files = order.map(
- (idx) => `static/features/feature_${idx}${suffix}.png`,
- );
-
- cp.execSync(
- `montage -mode concatenate -background none -tile 5x ${montage_files.join(
- " ",
- )} static/features/montage${is_dark ? "_dark" : "_light"}.png`,
- );
-}
-
-async function run() {
- if (
- !fs.existsSync("static/features") ||
- fs.readdirSync("static/features").length === 0
- ) {
- console.log("Generating feature screenshots!");
- fs.mkdirSync(path.join(__dirname, "static/features"), {
- recursive: true,
- });
-
- const x = await perspective;
- const server = new x.WebSocketServer({
- assets: [
- path.join(__dirname, "."),
- path.join(__dirname, "../node_modules"),
- ],
- });
-
- const indices = Array.from({ length: EXAMPLES.length }, (_, i) => i);
- for (let i = indices.length - 1; i > 0; i--) {
- const j = Math.floor(Math.random() * (i + 1));
- [indices[i], indices[j]] = [indices[j], indices[i]];
- }
-
- const browser = await puppeteer.launch({ headless: true });
- const page = await browser.newPage();
- await run_with_theme(page, false, indices);
- await run_with_theme(page, true, indices);
- await page.close();
- await browser.close();
- await server.close();
-
- fs.writeFileSync(
- path.join(__dirname, "static/features/montage_map.json"),
- JSON.stringify({
- tile_width: 200,
- tile_height: 150,
- columns: 5,
- order: indices,
- }),
- );
- }
-
- if (!fs.existsSync("static/blocks")) {
- fs.mkdirSync("static/blocks");
- }
-
- const { dist_examples } = await import("../examples/blocks/index.mjs");
- await dist_examples(`${__dirname}/static/blocks`);
-}
-
-function template(is_dark) {
- return fs
- .readFileSync(path.join(__dirname, "template.html"))
- .toString()
- .replace("/css/pro.css", is_dark ? "/css/pro-dark.css" : "/css/pro.css")
- .trim();
-}
-
-run();
diff --git a/docs/build.projects.mjs b/docs/build.projects.mjs
new file mode 100644
index 0000000000..13db96f004
--- /dev/null
+++ b/docs/build.projects.mjs
@@ -0,0 +1,258 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
+// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors. ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+import { execSync } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+import puppeteer from "puppeteer";
+import { TextWriter, Uint8ArrayReader, ZipReader } from "@zip.js/zip.js";
+import { COLLAGE, collage } from "./build.collage.mjs";
+import { screenshotTheme } from "./build.screenshot.mjs";
+
+const [NODE_MAJOR] = process.versions.node.split(".").map(Number);
+if (NODE_MAJOR < 20) {
+ console.error(
+ `build.projects.mjs requires Node >= 20 (found ${process.version}) — ` +
+ "the dataset unzip step fails on older runtimes with " +
+ '"malloc is not a function". Run `nvm use 22` and rebuild.',
+ );
+
+ process.exit(1);
+}
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const OUT = path.join(__dirname, "static/projects");
+const DIST_OUT = path.join(__dirname, "dist/projects");
+const PORT = 8129;
+
+/** One pass per `prefers-color-scheme` value (see `build.screenshot.mjs`). */
+const THEMES = ["light", "dark"];
+
+const perspective = await import(
+ "@perspective-dev/client/dist/esm/perspective.node.js"
+);
+
+const DATA = path.join(__dirname, "static/data");
+const DIST_DATA = path.join(__dirname, "dist/data");
+
+const NYPD_URL =
+ "https://rawcdn.githack.com/new-york-civil-liberties-union/NYPD-Misconduct-Complaint-Database-Updated/f6cea944b347c96eb26b76323013640dff4b3d00/CCRB%20Complaint%20Database%20Raw%2004.28.2023.zip?min=1";
+
+const OLYMPICS_DATASET =
+ "heesoo37/120-years-of-olympic-history-athletes-and-results";
+
+const EVICTIONS_URL =
+ "https://data.sfgov.org/resource/5cei-gny5.csv?$limit=50000";
+
+const MOVIES_URL = "https://vega.github.io/editor/data/movies.json";
+
+const MOVIES_SCHEMA = {
+ Title: "string",
+ "US Gross": "float",
+ "Worldwide Gross": "float",
+ "US DVD Sales": "float",
+ "Production Budget": "float",
+ "Release Date": "date",
+ "MPAA Rating": "string",
+ "Running Time min": "integer",
+ Distributor: "string",
+ Source: "string",
+ "Major Genre": "string",
+ "Creative Type": "string",
+ Director: "string",
+ "Rotten Tomatoes Rating": "integer",
+ "IMDB Rating": "float",
+ "IMDB Votes": "integer",
+};
+
+async function csvToArrow(csv) {
+ const table = await perspective.default.table(csv);
+ const view = await table.view();
+ const arrow = new Uint8Array(await view.to_arrow());
+ await view.delete();
+ await table.delete();
+ return arrow;
+}
+
+async function csvZipToArrow(zipBytes) {
+ const zipReader = new ZipReader(new Uint8ArrayReader(zipBytes));
+ const entries = await zipReader.getEntries();
+ const csv = await entries[0].getData(new TextWriter());
+ await zipReader.close();
+ return csvToArrow(csv);
+}
+
+async function buildEvictionsArrow(out) {
+ const response = await fetch(EVICTIONS_URL);
+ if (!response.ok) {
+ throw new Error(`HTTP ${response.status} ${response.statusText}`);
+ }
+
+ fs.writeFileSync(out, await csvToArrow(await response.text()));
+}
+
+/**
+ * `movies.json` dates are "Jun 12 1998"-style, which the engine's date
+ * parser rejects — normalize to ISO. Component-wise, so the local-time
+ * `Date` parse cannot shift a day.
+ */
+function isoDate(value) {
+ if (!value) {
+ return null;
+ }
+
+ const date = new Date(value);
+ if (Number.isNaN(date.getTime())) {
+ return null;
+ }
+
+ const pad = (x) => String(x).padStart(2, "0");
+ return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`;
+}
+
+async function buildMoviesArrow(out) {
+ const response = await fetch(MOVIES_URL);
+ if (!response.ok) {
+ throw new Error(`HTTP ${response.status} ${response.statusText}`);
+ }
+
+ const rows = (await response.json()).map((row) => ({
+ ...row,
+ "Release Date": isoDate(row["Release Date"]),
+ }));
+
+ const table = await perspective.default.table(MOVIES_SCHEMA);
+ await table.update(rows);
+ const view = await table.view();
+ const arrow = new Uint8Array(await view.to_arrow());
+ await view.delete();
+ await table.delete();
+ fs.writeFileSync(out, arrow);
+}
+
+async function buildNypdArrow(out) {
+ const response = await fetch(NYPD_URL);
+ if (!response.ok) {
+ throw new Error(`HTTP ${response.status} ${response.statusText}`);
+ }
+
+ const bytes = new Uint8Array(await response.arrayBuffer());
+ fs.writeFileSync(out, await csvZipToArrow(bytes));
+}
+
+async function buildOlympicsArrow(out) {
+ execSync(`cd ${DATA} && kaggle datasets download -d ${OLYMPICS_DATASET}`, {
+ stdio: "inherit",
+ });
+
+ const zipPath = path.join(DATA, `${OLYMPICS_DATASET.split("/")[1]}.zip`);
+ try {
+ const bytes = new Uint8Array(fs.readFileSync(zipPath));
+ fs.writeFileSync(out, await csvZipToArrow(bytes));
+ } finally {
+ fs.rmSync(zipPath, { force: true });
+ }
+}
+
+async function prepareDataset(name, build) {
+ fs.mkdirSync(DATA, { recursive: true });
+ const cached = path.join(DATA, name);
+ if (!fs.existsSync(cached)) {
+ try {
+ await build(cached);
+ console.log(`Wrote ${name}`);
+ } catch (e) {
+ fs.rmSync(cached, { force: true });
+ console.warn(
+ ` ✗ ${name}: ${e.message ?? e} — its Projects will 404.`,
+ );
+
+ return;
+ }
+ }
+
+ fs.mkdirSync(DIST_DATA, { recursive: true });
+ fs.copyFileSync(cached, path.join(DIST_DATA, name));
+}
+
+function copyRecursive(src, dest) {
+ if (fs.statSync(src).isDirectory()) {
+ fs.mkdirSync(dest, { recursive: true });
+ for (const child of fs.readdirSync(src)) {
+ copyRecursive(path.join(src, child), path.join(dest, child));
+ }
+ } else {
+ fs.copyFileSync(src, dest);
+ }
+}
+
+async function run() {
+ fs.mkdirSync(OUT, { recursive: true });
+ if (!fs.existsSync(path.join(__dirname, "dist/index.js"))) {
+ console.error("Run `node build.config.mjs` first — dist/ is missing.");
+ process.exit(1);
+ }
+
+ await prepareDataset("olympics.arrow", buildOlympicsArrow);
+ await prepareDataset("nypdccrb.arrow", buildNypdArrow);
+ await prepareDataset("evictions.arrow", buildEvictionsArrow);
+ await prepareDataset("movies.arrow", buildMoviesArrow);
+
+ const server = new perspective.WebSocketServer({
+ port: PORT,
+ assets: [
+ path.join(__dirname, "dist"),
+ path.join(__dirname, "static"),
+ path.join(__dirname, "../node_modules"),
+ ],
+ });
+
+ const browser = await puppeteer.launch({ headless: true });
+ const page = await browser.newPage();
+ await page.goto(`http://localhost:${PORT}/index.html`, {
+ waitUntil: "networkidle2",
+ });
+
+ const ids = await page.evaluate(async () => {
+ while (!window.__projectIds) {
+ await new Promise((x) => setTimeout(x, 20));
+ }
+
+ return window.__projectIds();
+ });
+
+ console.log(`${ids.length} projects × ${THEMES.length} themes`);
+ const target = { out: OUT, port: PORT };
+ let wrote = 0;
+ let failed = 0;
+ for (const theme of THEMES) {
+ const result = await screenshotTheme(page, ids, theme, target);
+ wrote += result.wrote;
+ failed += result.failed;
+ try {
+ await collage(page, ids, theme, target);
+ } catch (e) {
+ console.warn(` ✗ ${theme}/${COLLAGE}: ${e.message ?? e}`);
+ }
+ }
+
+ await browser.close();
+ await server.close();
+ copyRecursive(OUT, DIST_OUT);
+ console.log(`Thumbnails: ${wrote} written, ${failed} failed.`);
+}
+
+run().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
diff --git a/docs/build.screenshot.mjs b/docs/build.screenshot.mjs
new file mode 100644
index 0000000000..b36029eef2
--- /dev/null
+++ b/docs/build.screenshot.mjs
@@ -0,0 +1,110 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
+// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors. ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+const VIEWPORT = { width: 800, height: 600 };
+
+const LOAD_TIMEOUT = 180_000;
+const STREAMING = (id) => id.startsWith("market-");
+
+async function screenshot(page, id, theme, { out, port }) {
+ const file = path.join(out, theme, `${id}.png`);
+ if (fs.existsSync(file)) {
+ return "skipped";
+ }
+
+ await page.goto(`http://localhost:${port}/index.html`, {
+ waitUntil: "networkidle2",
+ });
+
+ await page.evaluate(async () => {
+ while (!window.__loadProject) {
+ await new Promise((x) => setTimeout(x, 20));
+ }
+
+ globalThis.__PERSPECTIVE_SCREENSHOT__ = true;
+ });
+
+ await page.evaluate(() => {
+ document.querySelector(".sidebar__collapse")?.click();
+ });
+
+ await page.evaluate((project) => window.__loadProject(project), id);
+ await page.evaluate(async () => {
+ const viewer = document.querySelector("perspective-viewer");
+ while (viewer.getPanelNames().length === 0) {
+ await new Promise((x) => setTimeout(x, 50));
+ }
+
+ await viewer.restore({ settings: false });
+ await viewer.flush();
+ });
+
+ if (STREAMING(id)) {
+ await page.evaluate(async () => {
+ const viewer = document.querySelector("perspective-viewer");
+ const table = await viewer.getTable();
+ let last = -1;
+ let stable = 0;
+ while (stable < 2) {
+ const size = await table.size();
+ stable = size === last ? stable + 1 : 0;
+ last = size;
+ await new Promise((x) => setTimeout(x, 500));
+ }
+
+ await viewer.flush();
+ });
+ }
+
+ const stage = await page.$("perspective-viewer");
+ fs.writeFileSync(file, await stage.screenshot());
+ return "wrote";
+}
+
+export async function screenshotTheme(page, ids, theme, { out, port }) {
+ fs.mkdirSync(path.join(out, theme), { recursive: true });
+ await page.emulateMediaFeatures([
+ { name: "prefers-color-scheme", value: theme },
+ ]);
+
+ await page.setViewport(VIEWPORT);
+ let wrote = 0;
+ let failed = 0;
+ for (const id of ids) {
+ let timer;
+ try {
+ const result = await Promise.race([
+ screenshot(page, id, theme, { out, port }),
+ new Promise((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error("timed out")),
+ LOAD_TIMEOUT,
+ );
+ }),
+ ]);
+
+ if (result === "wrote") {
+ wrote++;
+ }
+ } catch (e) {
+ failed++;
+ console.warn(` ✗ ${theme}/${id}: ${e.message ?? e}`);
+ } finally {
+ clearTimeout(timer);
+ }
+ }
+
+ return { wrote, failed };
+}
diff --git a/docs/compose.yml b/docs/compose.yml
index 801c87a176..42bd08862d 100644
--- a/docs/compose.yml
+++ b/docs/compose.yml
@@ -14,3 +14,35 @@ services:
- serve
- --hostname
- "0.0.0.0"
+
+ # A local S3-compatible backend for developing the "Create Data Source"
+ # S3 form. DEVELOPMENT ONLY: it is reachable because the dev page is also
+ # `http://localhost`, so the mixed-content block that stops an HTTPS
+ # deployment from reaching a plain-HTTP endpoint does not apply here.
+ # MinIO is also more lenient than AWS about signatures, so a green MinIO
+ # is necessary but not sufficient.
+ minio:
+ container_name: minio
+ image: minio/minio
+ ports:
+ - 9000:9000
+ - 9001:9001
+ environment:
+ MINIO_ROOT_USER: perspective
+ MINIO_ROOT_PASSWORD: perspective
+ MINIO_API_CORS_ALLOW_ORIGIN: "http://localhost:8080"
+ command: server /data --console-address ":9001"
+
+ minio-init:
+ image: minio/mc
+ depends_on:
+ - minio
+ volumes:
+ - ${PWD}/../node_modules/superstore-arrow:/fixtures:ro
+ entrypoint: >
+ /bin/sh -c "
+ until mc alias set local http://minio:9000 perspective perspective; do sleep 1; done;
+ mc mb --ignore-existing local/perspective;
+ mc cp /fixtures/superstore.lz4.arrow local/perspective/arrow/superstore.arrow;
+ exit 0;
+ "
diff --git a/docs/deploy.mjs b/docs/deploy.mjs
index bc78605f0a..6a0444e010 100644
--- a/docs/deploy.mjs
+++ b/docs/deploy.mjs
@@ -19,12 +19,13 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = path.resolve(__dirname, "..");
const DIST = path.join(__dirname, "dist");
const STAGING = path.join(REPO_ROOT, "dist-gh-pages");
-const BRANCH = "gh-pages";
+const DEPLOY_REPO =
+ "https://github.com/perspective-dev/perspective-dev.github.io.git";
function git(args, opts = {}) {
return execFileSync("git", args, {
stdio: "inherit",
- cwd: REPO_ROOT,
+ cwd: STAGING,
...opts,
});
}
@@ -47,22 +48,20 @@ if (!fs.existsSync(DIST)) {
}
if (!fs.existsSync(STAGING)) {
- git(["worktree", "add", STAGING, BRANCH]);
+ git(["clone", DEPLOY_REPO, STAGING], { cwd: REPO_ROOT });
} else {
- git(["fetch", "origin", BRANCH]);
- git(["checkout", `origin/${BRANCH}`], { cwd: STAGING });
+ git(["fetch", "origin"]);
+ git(["reset", "--hard", "origin/HEAD"]);
}
-// Clear tracked + untracked content in the staging worktree, preserving
-// the worktree's `.git` link.
-git(["rm", "-rf", "--quiet", "--ignore-unmatch", "."], { cwd: STAGING });
-git(["clean", "-fdx"], { cwd: STAGING });
+git(["rm", "-rf", "--quiet", "--ignore-unmatch", "."]);
+git(["clean", "-fdx"]);
for (const entry of fs.readdirSync(DIST)) {
copyRecursive(path.join(DIST, entry), path.join(STAGING, entry));
}
-git(["add", "-A"], { cwd: STAGING });
+git(["add", "-A"]);
-console.log(`Staged dist/ onto ${BRANCH} at ${STAGING}`);
+console.log(`Staged dist/ onto ${DEPLOY_REPO} at ${STAGING}`);
console.log(`Review with \`git -C ${STAGING} status\`, then commit and push.`);
diff --git a/docs/md/api_reference.md b/docs/md/api_reference.md
index d91655b8d8..4d8131c5da 100644
--- a/docs/md/api_reference.md
+++ b/docs/md/api_reference.md
@@ -5,9 +5,9 @@ Perspective's complete API is hosted on `docs.rs`:
- Python API
- [`perspective`](https://perspective-dev.github.io/python/index.html)
- [`perspective.widget`](https://perspective-dev.github.io/python/perspective/widget.html)
- - [`perspective.handlers.aiohttp`](https://perspective-dev.github.io/python/perspective/handlers/aiohttp.htm)
- - [`perspective.handlers.starlette`](https://perspective-dev.github.io/python/perspective/handlers/starlett.htm)
- - [`perspective.handlers.tornado`](https://perspective-dev.github.io/python/perspective/handlers/tornado.htm)
+ - [`perspective.handlers.aiohttp`](https://perspective-dev.github.io/python/perspective/handlers/aiohttp.html)
+ - [`perspective.handlers.starlette`](https://perspective-dev.github.io/python/perspective/handlers/starlette.html)
+ - [`perspective.handlers.tornado`](https://perspective-dev.github.io/python/perspective/handlers/tornado.html)
- JavaScript API
- [`@perspective-dev/client` Browser](https://perspective-dev.github.io/browser/modules/src_ts_perspective.browser.ts.html)
- [`@perspective-dev/client` Node.js](https://perspective-dev.github.io/node/modules/src_ts_perspective.node.ts.html)
diff --git a/docs/md/how_to/javascript/save_restore.md b/docs/md/how_to/javascript/save_restore.md
index 9d44a41d4d..367835de8a 100644
--- a/docs/md/how_to/javascript/save_restore.md
+++ b/docs/md/how_to/javascript/save_restore.md
@@ -115,6 +115,99 @@ const workspace_token = await elem.saveWorkspace();
await elem.restoreWorkspace(workspace_token);
```
-A `saveWorkspace()` token is a `WorkspaceConfig` (`{ version, layout, panels,
-... }`), not a `ViewerConfig` — passing it to the single-panel `restore()` will
-_not_ restore the layout (its `panels`/`layout` keys are ignored).
+A `saveWorkspace()` token is a `WorkspaceConfig`
+(`{ version, layout, panels, ... }`), not a `ViewerConfig` — passing it to the
+single-panel `restore()` will _not_ restore the layout (its `panels`/`layout`
+keys are ignored).
+
+## Colors, palettes and gradients
+
+Per-column color styling lives in a panel's `columns_config`, keyed by column
+name, and every color-scale value is a string usable verbatim in CSS:
+
+| Kind | Value |
+| -------- | ----------------------------------------------------------------------------------- |
+| color | `"#rrggbb"` (`#rgb`, `rgb()` and `rgba()` are accepted on input) |
+| palette | `"linear-gradient(to right, #rrggbb, #rrggbb, …)"` — N colors, **no** positions |
+| gradient | `"linear-gradient(to right, #rrggbb 0%, #rrggbb 37.5%, …)"` — every stop positioned |
+
+Which reader applies is decided by the style control's kind (the datagrid's
+`fg_colors`/`bg_colors` and the charts' `gradient` are gradients; `palette` is a
+palette), never by inspecting the string — a position anywhere in a palette is
+rejected, while a gradient may omit positions on input (the CSS
+implicit-position rules fill them) and may carry any direction token, which is
+normalized to `to right`. Values equal to the plugin's default are not
+serialized.
+
+```javascript
+await viewer.restore({
+ plugin: "Datagrid",
+ columns_config: {
+ Profit: {
+ number_bg_mode: "gradient",
+ bg_colors: "linear-gradient(to right, #ff0000, #ffffff, #0000ff)",
+ },
+ },
+});
+```
+
+Any of these may instead be a reference to a CSS custom property of the same
+kind — `"var(--psp-user--color-
+
`,
+ )
+ .join("");
+
+ await page.setViewport({ width: COLLAGE_WIDTH, height });
+ await page.setContent(
+ `
+