From 19531fc532cc0eecb840964b7d5d2838fa9fa18b Mon Sep 17 00:00:00 2001 From: jieyuexing Date: Thu, 27 Aug 2026 09:39:18 +0800 Subject: [PATCH 1/2] feat(read): add bounded XLSX workbook support --- Cargo.lock | 69 +- Cargo.toml | 2 + README.md | 5 +- README.zh-CN.md | 5 +- crates/opsail-read/Cargo.toml | 2 + crates/opsail-read/README.md | 128 +- crates/opsail-read/XLSX_COMPATIBILITY.md | 97 + .../examples/xlsx_collaboration_benchmark.rs | 698 ++++ crates/opsail-read/src/error.rs | 18 + crates/opsail-read/src/lib.rs | 42 +- crates/opsail-read/src/model.rs | 81 +- crates/opsail-read/src/xlsx.rs | 2950 +++++++++++++++++ crates/opsail-read/tests/xlsx.rs | 709 ++++ crates/opsail/Cargo.toml | 1 + crates/opsail/src/machine.rs | 78 +- crates/opsail/src/main.rs | 136 +- crates/opsail/tests/xlsx_cli.rs | 194 ++ packages/node/README.md | 36 + packages/node/index.d.ts | 171 +- packages/node/src/client.js | 257 +- packages/node/test/protocol.test.js | 174 + packages/node/test/skill.test.js | 3 + scripts/benchmark-xlsx-corpus.mjs | 307 ++ scripts/benchmark-xlsx-read.mjs | 163 + skills/opsail/SKILL.md | 51 +- 25 files changed, 6331 insertions(+), 46 deletions(-) create mode 100644 crates/opsail-read/XLSX_COMPATIBILITY.md create mode 100644 crates/opsail-read/examples/xlsx_collaboration_benchmark.rs create mode 100644 crates/opsail-read/src/xlsx.rs create mode 100644 crates/opsail-read/tests/xlsx.rs create mode 100644 crates/opsail/tests/xlsx_cli.rs create mode 100755 scripts/benchmark-xlsx-corpus.mjs create mode 100755 scripts/benchmark-xlsx-read.mjs diff --git a/Cargo.lock b/Cargo.lock index 57ff42f..eb7ad34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,6 +94,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -379,6 +388,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "crypto-common" version = "0.1.7" @@ -436,6 +451,17 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1365,6 +1391,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "zip", ] [[package]] @@ -1394,6 +1421,7 @@ dependencies = [ "encoding_rs", "futures-util", "opsail-chrome", + "quick-xml", "reqwest", "rustls", "serde", @@ -1406,6 +1434,7 @@ dependencies = [ "unicode-segmentation", "url", "wiremock", + "zip", ] [[package]] @@ -1610,6 +1639,15 @@ dependencies = [ "windows", ] +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.46" @@ -2733,7 +2771,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3077,12 +3115,41 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 0c67471..2f254f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ libc = "0.2.183" predicates = "3.1.4" process-wrap = { version = "9.1.0", default-features = false, features = ["job-object", "kill-on-drop", "tokio1"] } reqwest = { version = "0.13.4", default-features = false, features = ["brotli", "charset", "gzip", "http2", "rustls-no-provider", "stream", "system-proxy", "zstd"] } +quick-xml = "0.39.4" ring = "0.17.14" rustls = { version = "0.23.42", default-features = false, features = ["ring", "std", "tls12"] } semver = "1.0.28" @@ -44,6 +45,7 @@ unicode-segmentation = "1.13.3" url = { version = "2.5.8", features = ["serde"] } wiremock = "0.6.5" windows = "0.62.2" +zip = { version = "2.4.2", default-features = false, features = ["deflate"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/README.md b/README.md index 990fd88..bc1a992 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,14 @@ Opsail is a modular native toolkit that gives software agents small, composable, ## Core capabilities -### Read HTML +### Read HTML and XLSX -`opsail read` turns static HTML or a browser-rendered DOM into readable Markdown, sanitized HTML, or versioned JSON. It accepts URLs, files, stdin, an Opsail-owned isolated Chrome process, or an explicitly borrowed CDP endpoint. +`opsail read` turns static HTML or a browser-rendered DOM into readable Markdown, sanitized HTML, or versioned JSON. It also reads bounded sparse ranges from local XLSX workbooks. HTML accepts URLs, files, stdin, an Opsail-owned isolated Chrome process, or an explicitly borrowed CDP endpoint. ```sh opsail read https://example.com/article opsail read https://example.com/app --launch +opsail read ./book.xlsx --max-bytes 67108864 --range 'Summary!A1:H30' --format json ``` See [`opsail-read`](https://github.com/lencx/opsail/blob/main/crates/opsail-read/README.md) for acquisition, extraction, result contracts, and Rust APIs. See [`opsail-chrome`](https://github.com/lencx/opsail/blob/main/crates/opsail-chrome/README.md) for Chrome discovery, owned launch, borrowed CDP, navigation, and rendered DOM capture. diff --git a/README.zh-CN.md b/README.zh-CN.md index eaaf914..105072e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -24,13 +24,14 @@ Opsail 是一个模块化原生工具集，通过统一的命令行入口，为 ## 核心能力 -### 读取 HTML +### 读取 HTML 与 XLSX -`opsail read` 可以将静态 HTML 或浏览器渲染后的 DOM 转换为易读的 Markdown、经过清理的 HTML 或带版本的 JSON。输入可以来自 URL、文件、stdin、由 Opsail 启动的隔离 Chrome，或显式借用的 CDP 端点。 +`opsail read` 可以将静态 HTML 或浏览器渲染后的 DOM 转换为易读的 Markdown、经过清理的 HTML 或带版本的 JSON，也可以稀疏读取本地 XLSX 的受控范围。HTML 输入可以来自 URL、文件、stdin、由 Opsail 启动的隔离 Chrome，或显式借用的 CDP 端点。 ```sh opsail read https://example.com/article opsail read https://example.com/app --launch +opsail read ./book.xlsx --max-bytes 67108864 --range 'Summary!A1:H30' --format json ``` 内容获取、正文提取、结果契约和 Rust API 请参阅 [`opsail-read`](https://github.com/lencx/opsail/blob/main/crates/opsail-read/README.md)；Chrome 发现、自有启动、借用 CDP、页面导航和渲染 DOM 捕获请参阅 [`opsail-chrome`](https://github.com/lencx/opsail/blob/main/crates/opsail-chrome/README.md)。 diff --git a/crates/opsail-read/Cargo.toml b/crates/opsail-read/Cargo.toml index 527bb9f..d1e3e0a 100644 --- a/crates/opsail-read/Cargo.toml +++ b/crates/opsail-read/Cargo.toml @@ -16,6 +16,7 @@ dom_smoothie.workspace = true encoding_rs.workspace = true futures-util.workspace = true opsail-chrome = { path = "../opsail-chrome", version = "0.1.0" } +quick-xml.workspace = true reqwest.workspace = true rustls.workspace = true serde.workspace = true @@ -25,6 +26,7 @@ tokio.workspace = true tracing.workspace = true unicode-segmentation.workspace = true url.workspace = true +zip.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/opsail-read/README.md b/crates/opsail-read/README.md index d2abaf5..3718f33 100644 --- a/crates/opsail-read/README.md +++ b/crates/opsail-read/README.md @@ -1,10 +1,10 @@ # opsail-read `opsail-read` is the Rust library behind -[`opsail read`](https://github.com/lencx/opsail#read-html). It acquires static HTML -or delegates rendered DOM capture to `opsail-chrome`, extracts the primary -document, sanitizes the result, and returns a versioned `ReadResult` suitable -for agents and other programmatic callers. +[`opsail read`](https://github.com/lencx/opsail#read-html). It acquires static +HTML or delegates rendered DOM capture to `opsail-chrome`, and it sparsely reads +bounded regions from local XLSX workbooks. Both paths return versioned artifacts +suitable for agents and other programmatic callers. The extraction pipeline is browser-independent. `opsail-read` owns source validation, non-browser acquisition, extraction, sanitization, and result @@ -23,6 +23,17 @@ browser path. capture one page, and clean up the owned process and profile. - Resolve relative links and assets against a validated HTTP(S) base URL. - Extract readable Markdown and sanitized HTML with structured metadata. +- Read local XLSX sheet manifests and repeated `Sheet!A1:D20` ranges without + allocating from declared worksheet dimensions. +- Parse shared and inline strings, stored values, formulas and cached results, + supported date/time number formats, merged ranges, hidden state, and defined + names from OOXML. +- Inventory hyperlinks, filters, tables, conditional formats, validations, + page setup, outlines, drawings, charts, images, comments, controls, + sparklines, themes, and VBA-project presence without executing active content. +- Compare ZIP-part revisions without expanding OOXML, refresh changed worksheet + selections through `WorkbookSession`, and merge generated Markdown blocks + without overwriting agent-authored text. - Report source, extraction method, quality signals, and warnings through one stable result model. - Enforce byte, DOM element, nesting-depth, redirect, and timeout limits. @@ -60,6 +71,95 @@ acquisition, leaving `user_agent` as `None` sends `opsail/`; WeChat article URLs retain their browser-compatible automatic HTTP profile with an `opsail/` product token. An explicit value always wins. +## Read bounded XLSX ranges + +Use `read_artifact` for auto-detection. A local file ending in `.xlsx` returns +`ReadArtifact::Workbook`; other inputs preserve the existing +`ReadArtifact::Document` path: + +```rust +use opsail_read::{ReadArtifact, ReadOptions, ReadSource, read_artifact}; + +async fn inspect(path: std::path::PathBuf) -> Result<(), Box> { + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec![ + "Summary!A1:H30".to_owned(), + "'API Sheet'!B6:AY40".to_owned(), + ]; + let artifact = read_artifact(ReadSource::File(path), &options).await?; + if let ReadArtifact::Workbook(result) = artifact { + println!("{}", serde_json::to_string_pretty(&result)?); + } + Ok(()) +} +``` + +Repeated ranges are planned before extraction. The archive, workbook metadata, +shared strings, and styles are read once, and each selected worksheet XML part +is scanned once. Cells are stored sparsely by reference; a declared dimension +such as `A1:XFD1048576` never causes rectangular allocation. + +With no explicit ranges, visible sheets receive a bounded preview controlled by +`preview_rows` and `preview_columns`; hidden sheets remain in the manifest. +`max_cells` bounds published non-empty cells and marks affected selections as +truncated. `max_expanded_bytes` limits cumulative uncompressed OOXML read from +the ZIP package. The normal `max_bytes` limit still applies to the compressed +file itself. Stdout adapters can call `WorkbookReadResult::truncate_published_cells` +and rerender a stable prefix when their serialized transport has a stricter byte +budget than the cell-count limit. + +Formula expressions and cached values are reported separately. Opsail does not +recalculate formulas, so callers must not describe the cached value as current. +Legacy `.xls`, encrypted workbooks, macros, charts, images, conditional-format +rendering, and Excel-accurate visual layout are outside this reader. + +The read-side feature levels, completeness labels, human/agent mirror contract, +and 80% efficiency gate are specified in +[`XLSX_COMPATIBILITY.md`](XLSX_COMPATIBILITY.md). + +## Refresh a human-owned workbook without losing agent Markdown + +`WorkbookSession` is the in-process collaboration API. Keep the XLSX as the +human-owned source and keep agent analysis outside the generated marker blocks: + +```rust +use opsail_read::{ReadOptions, WorkbookSession, merge_markdown_mirror}; + +fn refresh( + path: std::path::PathBuf, + mut mirror: String, +) -> Result> { + let mut session = WorkbookSession::open(path, ReadOptions::default())?; + if mirror.is_empty() { + mirror = session.result().content.clone(); + } + + // The agent can append or edit text outside opsail:xlsx-generated blocks. + mirror.push_str("\n\n## Agent analysis\n\nPending human confirmation.\n"); + let refresh = session.refresh()?; + Ok(merge_markdown_mirror(&mirror, refresh.result)?) +} +``` + +An unchanged file uses a metadata fast path and expands zero OOXML bytes. A +worksheet-only change reuses cached shared strings/styles, parses only the +previous selections on changed sheets, and incrementally replaces generated +Markdown/HTML blocks. Workbook, shared-string, style, and theme changes take a +conservative full-refresh path. A full refresh is a correctness fallback, not +an efficiency failure hidden as a partial read. + +Run the collaboration benchmark against a workbook tree with: + +```sh +cargo +1.97.0 run --release -p opsail-read \ + --example xlsx_collaboration_benchmark -- \ + --md-rounds 4 --edit-samples 100 /path/to/workbooks +``` + +The benchmark never changes source files. It edits numeric, non-formula cells +only in temporary copies, compares every incremental selection with a cold +read, and verifies that agent-authored Markdown survives. + ## Process caller-captured HTML Browser hosts should capture the rendered HTML and final page URL themselves, @@ -206,7 +306,12 @@ response URL must also match the captured final main document. ## Result contract -Every successful entry point returns `ReadResult`, which contains: +`read` preserves the existing HTML-only `ReadResult`. `read_artifact` returns +the untagged `ReadArtifact` union: existing documents keep their version 1 +shape, while workbooks carry `artifactKind: "workbook"` and their own version 1 +shape. CLI and Node callers use the latter auto-detected contract. + +An HTML `ReadResult` contains: - `schema_version`: version of the serialized result contract. - `content` and `content_html`: readable Markdown and sanitized HTML. @@ -220,6 +325,19 @@ Every successful entry point returns `ReadResult`, which contains: Serialized fields use camel case, including `schemaVersion` and `contentHtml`. Callers should branch on structured fields rather than warning or error text. +A `WorkbookReadResult` contains common `content`, `contentHtml`, `metadata`, +`source`, and `warnings` fields plus: + +- `artifactKind: "workbook"`. +- `extraction.method: "ooxml-sparse"`. +- `workbook.sheets`: visibility, declared dimensions, scanned semantic bounds, + merge and hidden-row/column metadata. +- `workbook.definedNames`: scope, reference, and explicit `#REF!` validity. +- `workbook.selections`: requested/resolved bounds, sparse cell records, and + truncation. +- `workbook.statistics`: archive, expanded-byte, scan, style-only, and returned + cell counts. + ## Trust boundary Treat source HTML and all extracted metadata as untrusted input. `opsail-read` diff --git a/crates/opsail-read/XLSX_COMPATIBILITY.md b/crates/opsail-read/XLSX_COMPATIBILITY.md new file mode 100644 index 0000000..45bfa99 --- /dev/null +++ b/crates/opsail-read/XLSX_COMPATIBILITY.md @@ -0,0 +1,97 @@ +# XLSX read-side compatibility and efficiency contract + +This contract translates the persisted workbook features listed by +`rust_xlsxwriter` 0.99 into read-side observability. `rust_xlsxwriter` itself is +write-only; Opsail does not copy its API or claim that a structural inventory +is an Excel renderer. + +## Capability levels + +- **Semantic**: Opsail publishes the value or relationship an agent needs. +- **Structural**: Opsail reports bounded presence, count, range, or package-part + evidence, but does not reconstruct the feature visually. +- **Unsupported**: Opsail cannot currently establish the feature from the + workbook. Unsupported or partial behavior must remain explicit. + +| rust_xlsxwriter persisted feature | Opsail read level | Current evidence | +| --- | --- | --- | +| Basic Excel data types | Semantic | Strings, numbers, booleans, errors, blanks, and stored dates | +| Cell formatting | Structural | Style index, number format, format counts; fonts/fills/borders are not resolved yet | +| Formulas | Semantic | Expression, cached value, normal/shared/array/data-table kind, shared index and formula range; no recalculation | +| Charts | Structural | Package chart and drawing-part counts | +| Hyperlinks | Semantic | Cell reference, internal location or relationship target, display text, external flag | +| Page/printing setup | Structural | Page-setup/header-footer presence plus print-area/title defined names | +| Merged ranges | Semantic when the worksheet scan is complete | Exact merge references | +| Conditional formatting | Structural | Bounded `sqref` list and rule count | +| Data validation | Structural | Bounded `sqref` list and rule count | +| Cell notes | Structural | Comment-part and legacy-drawing counts | +| Textboxes | Structural | Drawing presence only; textbox text is not extracted | +| Checkboxes | Structural | Worksheet control and control-property part counts | +| Sparklines | Structural | Sparkline element count | +| Images | Structural | Media and drawing-part counts; pixels and anchors are not rendered | +| Workbook themes | Structural | Theme-part count | +| Rich multi-format strings | Semantic text, structural runs | Concatenated base text plus a `richText` flag; East Asian `rPh` pronunciation hints are excluded from cell text and per-run formatting is not published | +| Outline groupings | Semantic | Outlined row/column counts and maximum levels | +| Defined names | Semantic | Name, scope, reference, and invalid `#REF!` signal | +| Autofilters | Semantic | Exact worksheet filter range | +| Worksheet tables | Structural | Workbook and worksheet table-part counts | +| Macros | Structural safety signal | VBA project part count; code is never executed or published | + +Serde serialization and constant-memory writing are writer implementation +mechanisms rather than persisted workbook semantics, so they are not read-side +capability rows. + +## Completeness labels + +Default preview scans every worksheet to build a complete workbook feature +inventory, while publishing cells only from visible-sheet preview ranges. +Explicit range reads may stop after the last requested row. They set +`semanticBoundsComplete=false`, `features.complete=false`, and emit a warning +when the tail was not scanned. Declared worksheet dimensions never authorize a +dense allocation. + +## Human/agent collaboration loop + +The XLSX remains the human-owned source. Markdown is an agent-owned mirror with +replaceable generated blocks and unrestricted text outside those blocks. + +1. `WorkbookSession::open` performs one bounded cold read and retains the + generated selections, decoded shared strings, and styles. +2. An unchanged file-metadata stamp returns the cached result without opening + worksheet XML. When the stamp changes, a central-directory revision compares + each part's CRC32 and expanded size without expanding OOXML. Compressed size + is a metric, not content identity, because Excel may repack unchanged parts. +3. No XLSX content change returns the cached result with zero expanded bytes. +4. A worksheet-only change uses the already-open package, refreshes the prior + selections for changed sheets, reuses unchanged selections, and replaces + only the corresponding generated Markdown/HTML blocks. +5. Workbook, shared-string, style, or theme changes trigger a conservative full + refresh. +6. `merge_markdown_mirror` replaces only Opsail generated blocks. It preserves + agent-authored text byte-for-byte and rejects malformed/nested markers. + +## Performance acceptance + +Compatibility and efficiency are separate metrics. + +- Compatibility rate = successful cold, revision, and targeted reads divided + by eligible non-temporary XLSX files. +- Expanded-byte saving = `1 - incremental expanded bytes / cold expanded bytes`. +- Read-time saving = `1 - incremental refresh wall time / cold-read wall time`. +- A collaboration read is **high efficiency** only when both savings are at + least 80% and its refreshed selection matches a cold-read semantic projection. +- The agent-Markdown gate runs repeated unchanged refreshes across every valid + workbook. The direct human-edit gate uses size-stratified real workbook + samples, edits one visible numeric non-formula cell in a temporary copy, and + requires at least 80% of samples to meet the high-efficiency definition. The + declared repeated-Markdown plus one-human-edit cycle is reported separately. + Concurrent throughput cannot turn a slow individual read into a pass. + +Changing shared strings, styles, themes, or the workbook manifest is an +intentional negative example: it can affect many cells and therefore must take +the full-refresh path. The direct worksheet-only efficiency number must not be +generalized to those dependency-wide edits. + +The corpus benchmark never writes the source workbooks and publishes no cell +content. Human-edit simulations operate only on temporary sanitized fixtures or +temporary workbook copies. diff --git a/crates/opsail-read/examples/xlsx_collaboration_benchmark.rs b/crates/opsail-read/examples/xlsx_collaboration_benchmark.rs new file mode 100644 index 0000000..3c0c365 --- /dev/null +++ b/crates/opsail-read/examples/xlsx_collaboration_benchmark.rs @@ -0,0 +1,698 @@ +use std::error::Error; +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use opsail_read::{ + CellValueType, ReadOptions, SpreadsheetReadOptions, WorkbookReadResult, WorkbookSession, + merge_markdown_mirror, +}; +use quick_xml::events::{BytesText, Event}; +use quick_xml::{Reader, Writer}; +use serde_json::{Value, json}; +use tempfile::tempdir; +use zip::write::SimpleFileOptions; +use zip::{CompressionMethod, ZipArchive, ZipWriter}; + +const DEFAULT_MD_ROUNDS: usize = 4; +const DEFAULT_EDIT_SAMPLES: usize = 100; +const HIGH_EFFICIENCY_THRESHOLD: f64 = 0.8; + +#[derive(Debug)] +struct Arguments { + root: PathBuf, + md_rounds: usize, + edit_samples: usize, + max_files: Option, +} + +#[derive(Debug)] +struct AgentRefreshResult { + cold_micros: u64, + cold_bytes: usize, + refresh_micros: u64, + refresh_bytes: usize, + byte_saving: f64, + time_saving: f64, + markdown_preserved: bool, + semantic_equivalent: bool, +} + +#[derive(Debug)] +struct HumanEditResult { + path: PathBuf, + cold_micros: u64, + refresh_micros: u64, + probe_micros: u64, + extraction_micros: u64, + direct_byte_saving: f64, + direct_time_saving: f64, + cycle_byte_saving: f64, + cycle_time_saving: f64, + semantic_equivalent: bool, + markdown_preserved: bool, + full_refresh: bool, + changed_parts: Vec, +} + +fn main() -> Result<(), Box> { + let arguments = parse_arguments()?; + let mut workbooks = find_workbooks(&arguments.root)?; + if let Some(max_files) = arguments.max_files { + workbooks.truncate(max_files); + } + if workbooks.is_empty() { + return Err("no non-temporary XLSX workbooks found".into()); + } + + let options = benchmark_options(); + let started = Instant::now(); + let mut valid_paths = Vec::new(); + let mut agent_results = Vec::new(); + let mut failures = Vec::new(); + for path in &workbooks { + match benchmark_agent_refresh(path, &options, arguments.md_rounds) { + Ok(result) => { + valid_paths.push(path.clone()); + agent_results.push(result); + } + Err(error) => failures.push(json!({ + "path": relative_display(&arguments.root, path), + "stage": "agent-md-refresh", + "diagnostic": error.to_string(), + })), + } + } + + let human_paths = evenly_spaced_by_size(&valid_paths, arguments.edit_samples)?; + let mut human_results = Vec::new(); + let mut human_skips = Vec::new(); + for path in human_paths { + match benchmark_human_edit(&path, &options, arguments.md_rounds) { + Ok(Some(result)) => human_results.push(result), + Ok(None) => human_skips.push(json!({ + "path": relative_display(&arguments.root, &path), + "reason": "visible preview contains no editable numeric non-formula cell", + })), + Err(error) => failures.push(json!({ + "path": relative_display(&arguments.root, &path), + "stage": "human-xlsx-edit", + "diagnostic": error.to_string(), + })), + } + } + + let output = build_output( + &arguments, + &workbooks, + &agent_results, + &human_results, + &human_skips, + &failures, + started.elapsed(), + ); + println!("{}", serde_json::to_string_pretty(&output)?); + Ok(()) +} + +fn parse_arguments() -> Result> { + let mut values = std::env::args().skip(1).collect::>(); + let mut md_rounds = DEFAULT_MD_ROUNDS; + let mut edit_samples = DEFAULT_EDIT_SAMPLES; + let mut max_files = None; + let mut index = 0; + while index < values.len() { + let parsed = match values[index].as_str() { + "--md-rounds" => Some((&mut md_rounds, "--md-rounds")), + "--edit-samples" => Some((&mut edit_samples, "--edit-samples")), + "--max-files" => { + let value = values + .get(index + 1) + .ok_or("--max-files requires a value")? + .parse::()?; + if value == 0 { + return Err("--max-files must be greater than zero".into()); + } + max_files = Some(value); + values.drain(index..=index + 1); + continue; + } + _ => None, + }; + if let Some((destination, flag)) = parsed { + let value = values + .get(index + 1) + .ok_or_else(|| format!("{flag} requires a value"))? + .parse::()?; + if value == 0 { + return Err(format!("{flag} must be greater than zero").into()); + } + *destination = value; + values.drain(index..=index + 1); + } else { + index += 1; + } + } + if values.len() != 1 { + return Err( + "usage: xlsx_collaboration_benchmark [--md-rounds N] [--edit-samples N] [--max-files N] ROOT" + .into(), + ); + } + Ok(Arguments { + root: PathBuf::from(values.remove(0)).canonicalize()?, + md_rounds, + edit_samples, + max_files, + }) +} + +fn benchmark_options() -> ReadOptions { + ReadOptions { + max_bytes: 64 * 1024 * 1024, + spreadsheet: SpreadsheetReadOptions { + max_expanded_bytes: 512 * 1024 * 1024, + max_cells: 1_000, + preview_rows: 24, + preview_columns: 16, + ..SpreadsheetReadOptions::default() + }, + ..ReadOptions::default() + } +} + +fn find_workbooks(root: &Path) -> Result, Box> { + let mut directories = vec![root.to_path_buf()]; + let mut workbooks = Vec::new(); + while let Some(directory) = directories.pop() { + for entry in std::fs::read_dir(directory)? { + let entry = entry?; + let file_type = entry.file_type()?; + if file_type.is_dir() { + directories.push(entry.path()); + } else if file_type.is_file() && is_workbook(&entry.path()) { + workbooks.push(entry.path()); + } + } + } + workbooks.sort(); + Ok(workbooks) +} + +fn is_workbook(path: &Path) -> bool { + let temporary = path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("~$")); + !temporary + && path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("xlsx")) +} + +fn benchmark_agent_refresh( + path: &Path, + options: &ReadOptions, + md_rounds: usize, +) -> Result> { + let cold_started = Instant::now(); + let mut session = WorkbookSession::open(path.to_path_buf(), options.clone())?; + let cold_micros = elapsed_micros(cold_started.elapsed()); + let cold_bytes = session.result().workbook.statistics.expanded_bytes_read; + let expected_selections = selection_json(session.result())?; + let mut markdown = session.result().content.clone(); + let mut refresh_micros = 0_u64; + let mut refresh_bytes = 0_usize; + let mut markdown_preserved = true; + let mut semantic_equivalent = true; + for round in 0..md_rounds { + let note = format!("agent-manual-note-{round}"); + markdown.push_str(&format!("\n\n## Agent note {round}\n\n{note}\n")); + let refresh_started = Instant::now(); + let refresh = session.refresh()?; + refresh_micros = refresh_micros.saturating_add(elapsed_micros(refresh_started.elapsed())); + refresh_bytes = refresh_bytes.saturating_add(refresh.metrics.expanded_bytes_read); + markdown = merge_markdown_mirror(&markdown, refresh.result)?; + markdown_preserved &= markdown.contains(¬e); + semantic_equivalent &= selection_json(refresh.result)? == expected_selections; + if refresh.metrics.changed || !refresh.diff.unchanged { + return Err("unchanged workbook produced a changed revision".into()); + } + } + let baseline_bytes = cold_bytes.saturating_mul(md_rounds); + let baseline_micros = cold_micros.saturating_mul(md_rounds as u64); + Ok(AgentRefreshResult { + cold_micros, + cold_bytes, + refresh_micros, + refresh_bytes, + byte_saving: saving(baseline_bytes as f64, refresh_bytes as f64), + time_saving: saving(baseline_micros as f64, refresh_micros as f64), + markdown_preserved, + semantic_equivalent, + }) +} + +fn evenly_spaced_by_size( + paths: &[PathBuf], + requested: usize, +) -> Result, Box> { + if paths.is_empty() || requested == 0 { + return Ok(Vec::new()); + } + let mut sized = paths + .iter() + .map(|path| Ok((std::fs::metadata(path)?.len(), path.clone()))) + .collect::, std::io::Error>>()?; + sized.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); + let count = requested.min(sized.len()); + let mut selected = Vec::with_capacity(count); + for index in 0..count { + let position = ((2 * index + 1) * sized.len()) / (2 * count); + selected.push(sized[position.min(sized.len() - 1)].1.clone()); + } + Ok(selected) +} + +fn benchmark_human_edit( + source: &Path, + options: &ReadOptions, + md_rounds: usize, +) -> Result, Box> { + let directory = tempdir()?; + let path = directory.path().join("collaboration.xlsx"); + std::fs::copy(source, &path)?; + let cold_started = Instant::now(); + let mut session = WorkbookSession::open(path.clone(), options.clone())?; + let initial_cold_micros = elapsed_micros(cold_started.elapsed()); + let initial_cold_bytes = session.result().workbook.statistics.expanded_bytes_read; + let Some((sheet_part, cell_reference, replacement)) = editable_numeric_cell(session.result()) + else { + return Ok(None); + }; + let mut markdown = format!( + "{}\n\n## Agent analysis\n\nmanual-analysis-must-survive\n", + session.result().content + ); + let mut agent_refresh_micros = 0_u64; + let mut agent_refresh_bytes = 0_usize; + for round in 0..md_rounds { + let note = format!("human-cycle-agent-note-{round}"); + markdown.push_str(&format!("\n{note}\n")); + let started = Instant::now(); + let refresh = session.refresh()?; + agent_refresh_micros = + agent_refresh_micros.saturating_add(elapsed_micros(started.elapsed())); + agent_refresh_bytes = + agent_refresh_bytes.saturating_add(refresh.metrics.expanded_bytes_read); + markdown = merge_markdown_mirror(&markdown, refresh.result)?; + if !markdown.contains(¬e) { + return Err("agent-authored Markdown was lost before the human edit".into()); + } + } + + rewrite_numeric_cell(&path, &sheet_part, &cell_reference, &replacement)?; + let refresh_started = Instant::now(); + let refresh = session.refresh()?; + let refresh_micros = elapsed_micros(refresh_started.elapsed()); + markdown = merge_markdown_mirror(&markdown, refresh.result)?; + + let cold_after_started = Instant::now(); + let cold_after = WorkbookSession::open(path, options.clone())?; + let cold_after_micros = elapsed_micros(cold_after_started.elapsed()); + let cold_after_bytes = cold_after.result().workbook.statistics.expanded_bytes_read; + let semantic_equivalent = + selection_json(refresh.result)? == selection_json(cold_after.result())?; + let markdown_preserved = markdown.contains("manual-analysis-must-survive") + && (0..md_rounds) + .all(|round| markdown.contains(&format!("human-cycle-agent-note-{round}"))) + && refresh + .result + .workbook + .selections + .iter() + .flat_map(|selection| &selection.cells) + .any(|cell| cell.reference == cell_reference && cell.value == replacement); + + let cycle_baseline_bytes = initial_cold_bytes + .saturating_mul(md_rounds) + .saturating_add(cold_after_bytes); + let cycle_incremental_bytes = + agent_refresh_bytes.saturating_add(refresh.metrics.expanded_bytes_read); + let cycle_baseline_micros = initial_cold_micros + .saturating_mul(md_rounds as u64) + .saturating_add(cold_after_micros); + let cycle_incremental_micros = agent_refresh_micros.saturating_add(refresh_micros); + Ok(Some(HumanEditResult { + path: source.to_path_buf(), + cold_micros: cold_after_micros, + refresh_micros, + probe_micros: refresh.metrics.probe_duration_micros, + extraction_micros: refresh.result.extraction.duration_micros, + direct_byte_saving: saving( + cold_after_bytes as f64, + refresh.metrics.expanded_bytes_read as f64, + ), + direct_time_saving: saving(cold_after_micros as f64, refresh_micros as f64), + cycle_byte_saving: saving(cycle_baseline_bytes as f64, cycle_incremental_bytes as f64), + cycle_time_saving: saving( + cycle_baseline_micros as f64, + cycle_incremental_micros as f64, + ), + semantic_equivalent, + markdown_preserved, + full_refresh: refresh.metrics.full_refresh, + changed_parts: refresh.diff.changed_parts, + })) +} + +fn editable_numeric_cell(result: &WorkbookReadResult) -> Option<(String, String, String)> { + for selection in &result.workbook.selections { + let sheet = result + .workbook + .sheets + .iter() + .find(|sheet| sheet.name == selection.sheet)?; + for cell in &selection.cells { + if matches!(cell.value_type, CellValueType::Number) + && cell.formula_kind.is_none() + && let Ok(value) = cell.value.parse::() + && value.is_finite() + { + return Some(( + sheet.part.clone(), + cell.reference.clone(), + format_numeric_replacement(value + 1.0), + )); + } + } + } + None +} + +fn format_numeric_replacement(value: f64) -> String { + if value.fract() == 0.0 { + format!("{value:.0}") + } else { + value.to_string() + } +} + +fn rewrite_numeric_cell( + path: &Path, + target_part: &str, + cell_reference: &str, + replacement: &str, +) -> Result<(), Box> { + let file = File::open(path)?; + let mut archive = ZipArchive::new(file)?; + let rewritten_path = path.with_extension("opsail-rewrite.tmp"); + let rewritten_file = File::create(&rewritten_path)?; + let mut writer = ZipWriter::new(rewritten_file); + let mut replaced = false; + for index in 0..archive.len() { + let mut entry = archive.by_index(index)?; + let name = entry.name().to_owned(); + let is_directory = entry.is_dir(); + let compression = entry.compression(); + if !matches!( + compression, + CompressionMethod::Stored | CompressionMethod::Deflated + ) { + return Err(format!("unsupported ZIP compression method {compression:?}").into()); + } + let modified = entry.last_modified(); + let permissions = entry.unix_mode(); + let mut bytes = Vec::with_capacity(usize::try_from(entry.size())?); + entry.read_to_end(&mut bytes)?; + if name == target_part { + bytes = replace_cell_value(&bytes, cell_reference, replacement)?; + replaced = true; + } + let mut options = SimpleFileOptions::default().compression_method(compression); + if let Some(modified) = modified { + options = options.last_modified_time(modified); + } + if let Some(permissions) = permissions { + options = options.unix_permissions(permissions); + } + if is_directory { + writer.add_directory(name, options)?; + } else { + writer.start_file(name, options)?; + writer.write_all(&bytes)?; + } + } + writer.finish()?; + if !replaced { + return Err(format!("worksheet part `{target_part}` was not found").into()); + } + std::fs::rename(rewritten_path, path)?; + Ok(()) +} + +fn replace_cell_value( + xml: &[u8], + cell_reference: &str, + replacement: &str, +) -> Result, Box> { + let mut reader = Reader::from_reader(xml); + let mut writer = Writer::new(Vec::with_capacity(xml.len())); + let mut buffer = Vec::new(); + let mut in_target_cell = false; + let mut in_value = false; + let mut replaced = false; + loop { + let event = reader.read_event_into(&mut buffer)?; + match &event { + Event::Start(start) if start.local_name().as_ref() == b"c" => { + in_target_cell = start.attributes().filter_map(Result::ok).any(|attribute| { + attribute.key.local_name().as_ref() == b"r" + && attribute.value.as_ref() == cell_reference.as_bytes() + }); + writer.write_event(event.into_owned())?; + } + Event::Start(start) if in_target_cell && start.local_name().as_ref() == b"v" => { + in_value = true; + writer.write_event(event.into_owned())?; + } + Event::Text(_) if in_target_cell && in_value => { + writer.write_event(Event::Text(BytesText::new(replacement)))?; + replaced = true; + } + Event::End(end) if end.local_name().as_ref() == b"v" => { + in_value = false; + writer.write_event(event.into_owned())?; + } + Event::End(end) if end.local_name().as_ref() == b"c" => { + in_target_cell = false; + writer.write_event(event.into_owned())?; + } + Event::Eof => break, + _ => writer.write_event(event.into_owned())?, + } + buffer.clear(); + } + if !replaced { + return Err(format!("cell `{cell_reference}` did not contain a numeric value").into()); + } + Ok(writer.into_inner()) +} + +fn selection_json(result: &WorkbookReadResult) -> Result { + serde_json::to_string(&result.workbook.selections) +} + +fn build_output( + arguments: &Arguments, + workbooks: &[PathBuf], + agent_results: &[AgentRefreshResult], + human_results: &[HumanEditResult], + human_skips: &[Value], + failures: &[Value], + wall: Duration, +) -> Value { + let agent_efficient = agent_results + .iter() + .filter(|result| { + result.byte_saving >= HIGH_EFFICIENCY_THRESHOLD + && result.time_saving >= HIGH_EFFICIENCY_THRESHOLD + && result.markdown_preserved + && result.semantic_equivalent + }) + .count(); + let direct_human_efficient = human_results + .iter() + .filter(|result| { + result.direct_byte_saving >= HIGH_EFFICIENCY_THRESHOLD + && result.direct_time_saving >= HIGH_EFFICIENCY_THRESHOLD + && result.markdown_preserved + && result.semantic_equivalent + }) + .count(); + let cycle_efficient = human_results + .iter() + .filter(|result| { + result.cycle_byte_saving >= HIGH_EFFICIENCY_THRESHOLD + && result.cycle_time_saving >= HIGH_EFFICIENCY_THRESHOLD + && result.markdown_preserved + && result.semantic_equivalent + }) + .count(); + let agent_times = agent_results + .iter() + .map(|result| result.time_saving) + .collect::>(); + let agent_bytes = agent_results + .iter() + .map(|result| result.byte_saving) + .collect::>(); + let human_direct_times = human_results + .iter() + .map(|result| result.direct_time_saving) + .collect::>(); + let human_direct_bytes = human_results + .iter() + .map(|result| result.direct_byte_saving) + .collect::>(); + let cycle_times = human_results + .iter() + .map(|result| result.cycle_time_saving) + .collect::>(); + let cycle_bytes = human_results + .iter() + .map(|result| result.cycle_byte_saving) + .collect::>(); + json!({ + "schemaVersion": 1, + "benchmark": "opsail-xlsx-human-agent-collaboration", + "corpusRoot": arguments.root, + "workload": { + "agentMarkdownRoundsPerHumanXlsxEdit": arguments.md_rounds, + "humanEditRequestedSamples": arguments.edit_samples, + "humanEditActualSamples": human_results.len(), + "humanEditSkippedSamples": human_skips.len(), + "humanEditMutation": "one visible numeric non-formula cell in a temporary workbook copy", + }, + "acceptance": { + "thresholdPercent": HIGH_EFFICIENCY_THRESHOLD * 100.0, + "compatibility": { + "eligibleWorkbooks": workbooks.len(), + "successfulWorkbooks": agent_results.len(), + "actualPercent": percent(agent_results.len(), workbooks.len()), + }, + "agentMarkdownRefresh": { + "metric": "workbooks preserving Markdown and selections with both byte and time savings >= 80%", + "actualPercent": percent(agent_efficient, agent_results.len()), + "passed": percent(agent_efficient, agent_results.len()) >= 80.0, + }, + "directHumanXlsxRefresh": { + "metric": "edited samples equivalent to cold read with both byte and time savings >= 80%", + "actualPercent": percent(direct_human_efficient, human_results.len()), + "passed": percent(direct_human_efficient, human_results.len()) >= 80.0, + }, + "collaborationCycle": { + "metric": "declared repeated-MD plus one-XLSX-edit cycles with both byte and time savings >= 80%", + "actualPercent": percent(cycle_efficient, human_results.len()), + "passed": percent(cycle_efficient, human_results.len()) >= 80.0, + }, + }, + "performance": { + "wallMs": wall.as_secs_f64() * 1000.0, + "agentMarkdownRefresh": distribution(&agent_bytes, &agent_times), + "directHumanXlsxRefresh": distribution(&human_direct_bytes, &human_direct_times), + "collaborationCycle": distribution(&cycle_bytes, &cycle_times), + "agentColdReadMicros": duration_distribution(agent_results.iter().map(|result| result.cold_micros)), + "agentRefreshMicros": duration_distribution(agent_results.iter().map(|result| result.refresh_micros)), + "agentColdExpandedBytes": duration_distribution(agent_results.iter().map(|result| result.cold_bytes as u64)), + "agentRefreshExpandedBytes": duration_distribution(agent_results.iter().map(|result| result.refresh_bytes as u64)), + }, + "correctness": { + "agentMarkdownPreserved": agent_results.iter().filter(|result| result.markdown_preserved).count(), + "agentSemanticEquivalent": agent_results.iter().filter(|result| result.semantic_equivalent).count(), + "humanMarkdownPreserved": human_results.iter().filter(|result| result.markdown_preserved).count(), + "humanSemanticEquivalent": human_results.iter().filter(|result| result.semantic_equivalent).count(), + "humanFullRefreshes": human_results.iter().filter(|result| result.full_refresh).count(), + }, + "humanSamples": human_results.iter().map(|result| json!({ + "path": relative_display(&arguments.root, &result.path), + "coldMicros": result.cold_micros, + "refreshMicros": result.refresh_micros, + "probeMicros": result.probe_micros, + "extractionMicros": result.extraction_micros, + "directByteSavingPercent": result.direct_byte_saving * 100.0, + "directTimeSavingPercent": result.direct_time_saving * 100.0, + "cycleByteSavingPercent": result.cycle_byte_saving * 100.0, + "cycleTimeSavingPercent": result.cycle_time_saving * 100.0, + "semanticEquivalent": result.semantic_equivalent, + "markdownPreserved": result.markdown_preserved, + "fullRefresh": result.full_refresh, + "changedParts": result.changed_parts, + })).collect::>(), + "humanSkips": human_skips, + "failures": failures.iter().take(20).collect::>(), + "failureCount": failures.len(), + }) +} + +fn distribution(byte_savings: &[f64], time_savings: &[f64]) -> Value { + json!({ + "expandedByteSavingPercent": { + "p50": percentile(byte_savings, 0.50) * 100.0, + "p95": percentile(byte_savings, 0.95) * 100.0, + }, + "timeSavingPercent": { + "p50": percentile(time_savings, 0.50) * 100.0, + "p95": percentile(time_savings, 0.95) * 100.0, + }, + }) +} + +fn duration_distribution(values: impl Iterator) -> Value { + let values = values.map(|value| value as f64).collect::>(); + json!({ + "p50": percentile(&values, 0.50), + "p95": percentile(&values, 0.95), + }) +} + +fn percentile(values: &[f64], fraction: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + let index = ((sorted.len() as f64 * fraction).ceil() as usize) + .saturating_sub(1) + .min(sorted.len() - 1); + sorted[index] +} + +fn saving(baseline: f64, incremental: f64) -> f64 { + if baseline <= 0.0 { + 0.0 + } else { + 1.0 - incremental / baseline + } +} + +fn percent(numerator: usize, denominator: usize) -> f64 { + if denominator == 0 { + 0.0 + } else { + numerator as f64 / denominator as f64 * 100.0 + } +} + +fn elapsed_micros(duration: Duration) -> u64 { + u64::try_from(duration.as_micros()).unwrap_or(u64::MAX) +} + +fn relative_display(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .into_owned() +} diff --git a/crates/opsail-read/src/error.rs b/crates/opsail-read/src/error.rs index 62d79d7..f1b4035 100644 --- a/crates/opsail-read/src/error.rs +++ b/crates/opsail-read/src/error.rs @@ -25,6 +25,24 @@ pub enum ReadError { #[error("input does not appear to be an HTML document")] NotHtml, + #[error("input is not a valid XLSX workbook: {0}")] + InvalidXlsx(String), + + #[error("Markdown is not a valid Opsail XLSX mirror: {0}")] + InvalidMarkdownMirror(String), + + #[error("invalid spreadsheet range `{selector}`: {reason}")] + InvalidSpreadsheetRange { selector: String, reason: String }, + + #[error("spreadsheet worksheet `{0}` was not found")] + WorksheetNotFound(String), + + #[error("spreadsheet OOXML exceeds the {limit} expanded byte limit")] + SpreadsheetExpandedTooLarge { limit: usize }, + + #[error("spreadsheet extraction task failed")] + SpreadsheetTask, + #[error("unsupported response content type `{0}`")] UnsupportedContentType(String), diff --git a/crates/opsail-read/src/lib.rs b/crates/opsail-read/src/lib.rs index 34436eb..665225b 100644 --- a/crates/opsail-read/src/lib.rs +++ b/crates/opsail-read/src/lib.rs @@ -6,6 +6,7 @@ mod model; mod source; mod standardize; mod verification; +mod xlsx; use std::time::Instant; @@ -15,10 +16,47 @@ use url::Url; pub use error::ReadError; pub use model::{ CapturedDocument, DEFAULT_CONNECT_TIMEOUT, DEFAULT_MAX_BYTES, DEFAULT_MAX_DEPTH, - DEFAULT_MAX_ELEMENTS, DEFAULT_TIMEOUT, DocumentMetadata, ExtractionInfo, ExtractionMethod, - Input, QualityGrade, QualityInfo, ReadOptions, ReadResult, ReadSource, SourceInfo, SourceKind, + DEFAULT_MAX_ELEMENTS, DEFAULT_MAX_SPREADSHEET_CELLS, DEFAULT_MAX_SPREADSHEET_EXPANDED_BYTES, + DEFAULT_SPREADSHEET_PREVIEW_COLUMNS, DEFAULT_SPREADSHEET_PREVIEW_ROWS, DEFAULT_TIMEOUT, + DocumentMetadata, ExtractionInfo, ExtractionMethod, Input, QualityGrade, QualityInfo, + ReadArtifact, ReadOptions, ReadResult, ReadSource, SourceInfo, SourceKind, + SpreadsheetReadOptions, }; pub use opsail_chrome::{CdpSource, CdpWaitUntil, ChromeError, ChromeSource}; +pub use xlsx::{ + ArtifactKind, CellValueType, DateSystem, DefinedName, FormulaKind, SelectionBounds, SheetState, + WorkbookCell, WorkbookExtractionInfo, WorkbookExtractionMethod, WorkbookFeatureInventory, + WorkbookHyperlink, WorkbookInfo, WorkbookMetadata, WorkbookPartRevision, WorkbookReadResult, + WorkbookRevision, WorkbookRevisionDiff, WorkbookSelection, WorkbookSession, + WorkbookSessionMetrics, WorkbookSessionRefresh, WorkbookSheet, WorkbookStatistics, + WorksheetFeatureInventory, inspect_workbook_revision, merge_markdown_mirror, +}; + +/// Auto-detect and read a supported document artifact. +/// +/// Local files ending in `.xlsx` use the bounded sparse OOXML reader. Other +/// inputs preserve the existing HTML acquisition and extraction behavior. +pub async fn read_artifact( + source: ReadSource, + options: &ReadOptions, +) -> Result { + if let ReadSource::File(path) = &source + && xlsx::is_xlsx_path(path) + { + let path = path.clone(); + let options = options.clone(); + let workbook = tokio::task::spawn_blocking(move || xlsx::read_xlsx(path, &options)) + .await + .map_err(|_| ReadError::SpreadsheetTask)??; + return Ok(ReadArtifact::Workbook(workbook)); + } + if options.spreadsheet.revision_only { + return Err(ReadError::InvalidXlsx( + "revision-only reads require a local .xlsx file".to_owned(), + )); + } + read(source, options).await.map(ReadArtifact::Document) +} /// Acquire and extract one HTML document. pub async fn read(source: ReadSource, options: &ReadOptions) -> Result { diff --git a/crates/opsail-read/src/model.rs b/crates/opsail-read/src/model.rs index 15ee36c..309cb77 100644 --- a/crates/opsail-read/src/model.rs +++ b/crates/opsail-read/src/model.rs @@ -10,6 +10,10 @@ use url::Url; pub const DEFAULT_MAX_BYTES: usize = 5 * 1024 * 1024; pub const DEFAULT_MAX_ELEMENTS: usize = 50_000; pub const DEFAULT_MAX_DEPTH: usize = 256; +pub const DEFAULT_MAX_SPREADSHEET_CELLS: usize = 10_000; +pub const DEFAULT_MAX_SPREADSHEET_EXPANDED_BYTES: usize = 64 * 1024 * 1024; +pub const DEFAULT_SPREADSHEET_PREVIEW_ROWS: u32 = 24; +pub const DEFAULT_SPREADSHEET_PREVIEW_COLUMNS: u16 = 16; pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(15); pub const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); pub(crate) const DEFAULT_USER_AGENT: &str = concat!("opsail/", env!("CARGO_PKG_VERSION")); @@ -101,6 +105,43 @@ pub struct ReadOptions { /// An exact User-Agent value, or `None` to select the automatic profile. pub user_agent: Option, pub accept_language: Option, + /// XLSX range selection and resource limits. + pub spreadsheet: SpreadsheetReadOptions, +} + +/// Options for bounded, sparse XLSX extraction. +#[derive(Debug, Clone)] +pub struct SpreadsheetReadOptions { + /// Repeated `Sheet!A1:D20` selectors. With no selectors, visible sheets get + /// a bounded preview instead of an unbounded workbook dump. + pub ranges: Vec, + /// Maximum number of non-empty cell records published across selections. + pub max_cells: usize, + /// Maximum cumulative uncompressed OOXML bytes read from the ZIP package. + pub max_expanded_bytes: usize, + /// Preview row count used when no explicit ranges are supplied. + pub preview_rows: u32, + /// Preview column count used when no explicit ranges are supplied. + pub preview_columns: u16, + /// Include formula expressions alongside cached workbook values. + pub include_formulas: bool, + /// Return the central-directory revision and workbook manifest without + /// expanding shared strings, styles, or worksheet XML. + pub revision_only: bool, +} + +impl Default for SpreadsheetReadOptions { + fn default() -> Self { + Self { + ranges: Vec::new(), + max_cells: DEFAULT_MAX_SPREADSHEET_CELLS, + max_expanded_bytes: DEFAULT_MAX_SPREADSHEET_EXPANDED_BYTES, + preview_rows: DEFAULT_SPREADSHEET_PREVIEW_ROWS, + preview_columns: DEFAULT_SPREADSHEET_PREVIEW_COLUMNS, + include_formulas: true, + revision_only: false, + } + } } impl Default for ReadOptions { @@ -112,10 +153,10 @@ impl Default for ReadOptions { max_bytes: DEFAULT_MAX_BYTES, user_agent: None, accept_language: None, + spreadsheet: SpreadsheetReadOptions::default(), } } } - #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "kebab-case")] pub enum SourceKind { @@ -216,6 +257,44 @@ pub struct ReadResult { pub warnings: Vec, } +/// Auto-detected artifact returned by [`crate::read_artifact`]. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum ReadArtifact { + Document(ReadResult), + Workbook(crate::xlsx::WorkbookReadResult), +} + +impl ReadArtifact { + pub fn content(&self) -> &str { + match self { + Self::Document(result) => &result.content, + Self::Workbook(result) => &result.content, + } + } + + pub fn content_html(&self) -> &str { + match self { + Self::Document(result) => &result.content_html, + Self::Workbook(result) => &result.content_html, + } + } + + pub fn warnings(&self) -> &[String] { + match self { + Self::Document(result) => &result.warnings, + Self::Workbook(result) => &result.warnings, + } + } + + pub fn property(&self, name: &str) -> Option { + match self { + Self::Document(result) => result.property(name), + Self::Workbook(result) => result.property(name), + } + } +} + impl ReadResult { /// Return a named projection used by the CLI's `--property` option. pub fn property(&self, name: &str) -> Option { diff --git a/crates/opsail-read/src/xlsx.rs b/crates/opsail-read/src/xlsx.rs new file mode 100644 index 0000000..a1205f5 --- /dev/null +++ b/crates/opsail-read/src/xlsx.rs @@ -0,0 +1,2950 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs::File; +use std::io::{BufRead, BufReader, Read, Seek}; +use std::path::{Path, PathBuf}; +use std::time::Instant; +use std::time::SystemTime; + +use quick_xml::Reader; +use quick_xml::events::{BytesRef, BytesStart, Event}; +use serde::Serialize; +use serde_json::{Value, json}; +use url::Url; +use zip::ZipArchive; +use zip::result::ZipError; + +use crate::error::ReadError; +use crate::model::{ReadOptions, SourceInfo, SourceKind, SpreadsheetReadOptions}; + +const XLSX_CONTENT_TYPE: &str = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; +const PREVIEW_GRID_MAX_AREA: u64 = 2_000; +const MAX_CELL_TEXT_BYTES: usize = 32_767 * 4; +const MAX_FEATURE_REFERENCES: usize = 10_000; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ArtifactKind { + Workbook, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookReadResult { + pub schema_version: u8, + pub artifact_kind: ArtifactKind, + pub content: String, + pub content_html: String, + pub metadata: WorkbookMetadata, + pub source: SourceInfo, + pub extraction: WorkbookExtractionInfo, + pub revision: WorkbookRevision, + pub workbook: WorkbookInfo, + pub warnings: Vec, +} + +impl WorkbookReadResult { + pub fn property(&self, name: &str) -> Option { + let value = match name { + "content" | "markdown" => json!(self.content), + "contentHtml" | "html" => json!(self.content_html), + "title" => json!(self.metadata.title), + "source" => json!(self.source), + "extraction" => json!(self.extraction), + "revision" => json!(self.revision), + "workbook" => json!(self.workbook), + "sheets" => json!(self.workbook.sheets), + "selections" => json!(self.workbook.selections), + "definedNames" => json!(self.workbook.defined_names), + "features" => json!(self.workbook.features), + "statistics" => json!(self.workbook.statistics), + "metrics" => json!({ + "extraction": self.extraction, + "revisionId": self.revision.id, + "statistics": self.workbook.statistics, + "sheets": self.workbook.sheets.iter().map(|sheet| json!({ + "name": sheet.name, + "state": sheet.state, + "part": sheet.part, + })).collect::>(), + }), + _ => return None, + }; + Some(value) + } + + /// Retain at most `max_cells` published cells in stable selection order. + /// + /// This is used by stdout adapters that must fit a serialized workbook + /// result into a bounded transport. Worksheet scanning remains governed by + /// `SpreadsheetReadOptions::max_cells`; this method only trims the already + /// selected result and regenerates its Markdown and HTML mirrors. + pub fn truncate_published_cells(&mut self, max_cells: usize) -> usize { + let mut remaining = max_cells; + for selection in &mut self.workbook.selections { + if selection.cells.len() > remaining { + selection.cells.truncate(remaining); + selection.truncated = true; + } + remaining = remaining.saturating_sub(selection.cells.len()); + } + let retained = self + .workbook + .selections + .iter() + .map(|selection| selection.cells.len()) + .sum(); + self.workbook.statistics.returned_cells = retained; + self.content = render_markdown(&self.metadata.title, &self.workbook); + self.content_html = render_html(&self.metadata.title, &self.workbook); + retained + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookMetadata { + pub title: String, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookExtractionInfo { + pub method: WorkbookExtractionMethod, + pub duration_ms: u64, + pub duration_micros: u64, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum WorkbookExtractionMethod { + OoxmlSparse, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookInfo { + pub format: &'static str, + pub date_system: DateSystem, + pub sheets: Vec, + pub defined_names: Vec, + pub selections: Vec, + pub features: WorkbookFeatureInventory, + pub statistics: WorkbookStatistics, +} + +/// Workbook-level feature inventory. Package-part counts are complete without +/// expanding binary media. Worksheet-level inventory is complete only when +/// `inventory_complete` is true. +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookFeatureInventory { + pub inventory_complete: bool, + pub cell_formats: usize, + pub custom_number_formats: usize, + pub rich_string_items: usize, + pub theme_parts: usize, + pub drawing_parts: usize, + pub chart_parts: usize, + pub image_parts: usize, + pub table_parts: usize, + pub comment_parts: usize, + pub control_property_parts: usize, + pub macro_project_parts: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookRevision { + pub id: String, + pub compressed_bytes: u64, + pub expanded_bytes: u64, + pub parts: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookPartRevision { + pub name: String, + pub crc32: String, + pub compressed_bytes: u64, + pub expanded_bytes: u64, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookRevisionDiff { + pub unchanged: bool, + pub requires_full_refresh: bool, + pub changed_parts: Vec, + pub changed_worksheet_parts: Vec, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookSessionMetrics { + pub changed: bool, + pub full_refresh: bool, + pub probe_duration_micros: u64, + pub refresh_duration_micros: u64, + pub expanded_bytes_read: usize, +} + +#[derive(Debug)] +pub struct WorkbookSessionRefresh<'a> { + pub result: &'a WorkbookReadResult, + pub diff: WorkbookRevisionDiff, + pub metrics: WorkbookSessionMetrics, +} + +#[derive(Debug)] +pub struct WorkbookSession { + path: PathBuf, + options: ReadOptions, + result: WorkbookReadResult, + context: WorkbookSessionContext, + stamp: WorkbookFileStamp, +} + +#[derive(Debug)] +struct WorkbookSessionContext { + shared_strings: Vec, + styles: Styles, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct WorkbookFileStamp { + bytes: u64, + modified: Option, +} + +impl WorkbookRevision { + #[must_use] + pub fn diff(&self, current: &Self) -> WorkbookRevisionDiff { + let previous: BTreeMap<&str, &WorkbookPartRevision> = self + .parts + .iter() + .map(|part| (part.name.as_str(), part)) + .collect(); + let current_parts: BTreeMap<&str, &WorkbookPartRevision> = current + .parts + .iter() + .map(|part| (part.name.as_str(), part)) + .collect(); + let mut names: Vec<&str> = previous + .keys() + .chain(current_parts.keys()) + .copied() + .collect(); + names.sort_unstable(); + names.dedup(); + let changed_parts: Vec = names + .into_iter() + .filter(|name| match (previous.get(name), current_parts.get(name)) { + (Some(previous), Some(current)) => !same_part_content(previous, current), + (None, None) => false, + _ => true, + }) + .map(str::to_owned) + .collect(); + let changed_worksheet_parts = changed_parts + .iter() + .filter_map(|name| worksheet_part_for_change(name)) + .collect::>() + .into_iter() + .collect(); + let requires_full_refresh = changed_parts.iter().any(|name| { + matches!( + name.as_str(), + "[Content_Types].xml" + | "xl/workbook.xml" + | "xl/_rels/workbook.xml.rels" + | "xl/sharedStrings.xml" + | "xl/styles.xml" + ) || name.starts_with("xl/theme/") + }); + WorkbookRevisionDiff { + unchanged: changed_parts.is_empty(), + requires_full_refresh, + changed_parts, + changed_worksheet_parts, + } + } +} + +/// Compression ratios and ZIP offsets can change when Excel rewrites a +/// package even when an OOXML part is byte-for-byte identical. CRC32 plus the +/// expanded size identifies the part content for incremental refreshes without +/// turning a harmless package rewrite into a full workbook refresh. +fn same_part_content(previous: &WorkbookPartRevision, current: &WorkbookPartRevision) -> bool { + previous.crc32 == current.crc32 && previous.expanded_bytes == current.expanded_bytes +} + +impl WorkbookSession { + /// Open a workbook and retain its bounded result for revision-aware refreshes. + pub fn open(path: PathBuf, mut options: ReadOptions) -> Result { + options.spreadsheet.revision_only = false; + let stamp = WorkbookFileStamp::read(&path, options.max_bytes)?; + let (result, context) = read_xlsx_with_context(path.clone(), &options)?; + Ok(Self { + path, + options, + result, + context, + stamp, + }) + } + + #[must_use] + pub fn result(&self) -> &WorkbookReadResult { + &self.result + } + + /// Refresh changed worksheet selections while reusing unchanged selection + /// results. Workbook, shared-string, style, or theme changes fail over to a + /// full refresh because they can affect every sheet. + pub fn refresh(&mut self) -> Result, ReadError> { + let refresh_started = Instant::now(); + let probe_started = Instant::now(); + let stamp = WorkbookFileStamp::read(&self.path, self.options.max_bytes)?; + if stamp == self.stamp { + return Ok(WorkbookSessionRefresh { + result: &self.result, + diff: WorkbookRevisionDiff { + unchanged: true, + requires_full_refresh: false, + changed_parts: Vec::new(), + changed_worksheet_parts: Vec::new(), + }, + metrics: WorkbookSessionMetrics { + changed: false, + full_refresh: false, + probe_duration_micros: elapsed_micros(probe_started.elapsed()), + refresh_duration_micros: elapsed_micros(refresh_started.elapsed()), + expanded_bytes_read: 0, + }, + }); + } + let (package, revision) = open_workbook_package( + &self.path, + self.options.max_bytes, + self.options.spreadsheet.max_expanded_bytes, + )?; + let probe_duration_micros = elapsed_micros(probe_started.elapsed()); + let diff = self.result.revision.diff(&revision); + if diff.unchanged { + apply_revision(&mut self.result, revision); + self.stamp = stamp; + return Ok(WorkbookSessionRefresh { + result: &self.result, + diff, + metrics: WorkbookSessionMetrics { + changed: false, + full_refresh: false, + probe_duration_micros, + refresh_duration_micros: elapsed_micros(refresh_started.elapsed()), + expanded_bytes_read: 0, + }, + }); + } + + let mut full_refresh = diff.requires_full_refresh; + let mut ranges = Vec::new(); + for part in &diff.changed_worksheet_parts { + let Some(sheet) = self + .result + .workbook + .sheets + .iter() + .find(|sheet| sheet.part == *part) + else { + full_refresh = true; + break; + }; + let mut sheet_ranges = self + .result + .workbook + .selections + .iter() + .filter(|selection| selection.sheet == sheet.name) + .map(|selection| selection.requested.clone()) + .peekable(); + if sheet_ranges.peek().is_none() { + full_refresh = true; + break; + } + ranges.extend(sheet_ranges); + } + + let expanded_bytes_read; + if full_refresh { + drop(package); + let (result, context) = read_xlsx_with_context(self.path.clone(), &self.options)?; + self.context = context; + expanded_bytes_read = result.workbook.statistics.expanded_bytes_read; + self.result = result; + } else if ranges.is_empty() { + apply_revision(&mut self.result, revision); + expanded_bytes_read = 0; + } else { + let mut options = self.options.clone(); + options.spreadsheet.ranges = ranges; + read_xlsx_incremental( + package, + &self.path, + &options, + &mut self.result, + &self.context, + revision, + &diff.changed_worksheet_parts, + )?; + expanded_bytes_read = self.result.workbook.statistics.expanded_bytes_read; + } + self.stamp = stamp; + Ok(WorkbookSessionRefresh { + result: &self.result, + diff, + metrics: WorkbookSessionMetrics { + changed: true, + full_refresh, + probe_duration_micros, + refresh_duration_micros: elapsed_micros(refresh_started.elapsed()), + expanded_bytes_read, + }, + }) + } +} + +impl WorkbookFileStamp { + fn read(path: &Path, max_bytes: usize) -> Result { + let metadata = std::fs::metadata(path).map_err(|source| ReadError::ReadFile { + path: path.to_path_buf(), + source, + })?; + if !metadata.is_file() { + return Err(ReadError::NotRegularFile { + path: path.to_path_buf(), + }); + } + if metadata.len() > max_bytes as u64 { + return Err(ReadError::InputTooLarge { limit: max_bytes }); + } + Ok(Self { + bytes: metadata.len(), + modified: metadata.modified().ok(), + }) + } +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum DateSystem { + Excel1900, + Excel1904, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookSheet { + pub index: usize, + pub name: String, + pub part: String, + pub revision: WorkbookPartRevision, + pub state: SheetState, + #[serde(skip_serializing_if = "Option::is_none")] + pub declared_dimension: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub semantic_bounds: Option, + pub semantic_bounds_complete: bool, + pub selected: bool, + pub merged_ranges: Vec, + pub hidden_rows: usize, + pub hidden_columns: usize, + pub features: WorksheetFeatureInventory, +} + +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorksheetFeatureInventory { + pub scanned: bool, + pub complete: bool, + pub feature_references_truncated: bool, + pub formula_cells: usize, + pub hyperlinks: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub auto_filter: Option, + pub table_parts: usize, + pub drawing_parts: usize, + pub comment_drawing_parts: usize, + pub conditional_format_rules: usize, + pub conditional_format_ranges: Vec, + pub data_validation_rules: usize, + pub data_validation_ranges: Vec, + pub page_setup: bool, + pub header_footer: bool, + pub outlined_rows: usize, + pub outlined_columns: usize, + pub max_row_outline_level: u8, + pub max_column_outline_level: u8, + pub sparklines: usize, + pub controls: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookHyperlink { + pub reference: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub target: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub location: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub display: Option, + pub external: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum SheetState { + Visible, + Hidden, + VeryHidden, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DefinedName { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub local_sheet_index: Option, + pub reference: String, + pub valid_reference: bool, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookSelection { + pub requested: String, + pub sheet: String, + pub range: String, + pub bounds: SelectionBounds, + pub cells: Vec, + pub truncated: bool, +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SelectionBounds { + pub start_row: u32, + pub start_column: u16, + pub end_row: u32, + pub end_column: u16, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookCell { + pub reference: String, + pub row: u32, + pub column: u16, + pub value_type: CellValueType, + pub value: String, + pub display: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub formula: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub formula_kind: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub formula_reference: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub shared_formula_index: Option, + pub rich_text: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub style_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub number_format: Option, +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum CellValueType { + String, + Number, + Boolean, + Error, + Date, + Blank, +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum FormulaKind { + Normal, + Shared, + Array, + DataTable, + Other, +} + +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookStatistics { + pub archive_entries: usize, + pub expanded_bytes_read: usize, + pub scanned_sheets: usize, + pub cell_elements: usize, + pub non_empty_cells: usize, + pub style_only_cells: usize, + pub returned_cells: usize, +} + +#[derive(Debug)] +struct SheetDescriptor { + name: String, + state: SheetState, + path: String, +} + +#[derive(Debug)] +struct ParsedWorkbook { + date_system: DateSystem, + sheets: Vec, + defined_names: Vec, +} + +#[derive(Debug, Clone)] +struct SelectionPlan { + output_index: usize, + sheet_index: usize, + bounds: SelectionBounds, +} + +#[derive(Debug, Default)] +struct Styles { + custom_formats: HashMap, + cell_formats: Vec, +} + +impl Styles { + fn number_format(&self, style_index: Option) -> Option { + let id = *self.cell_formats.get(style_index?)?; + self.custom_formats + .get(&id) + .cloned() + .or_else(|| builtin_number_format(id).map(str::to_owned)) + } +} + +#[derive(Debug, Clone)] +struct SharedString { + text: String, + rich: bool, +} + +#[derive(Debug, Clone)] +struct PartRelationship { + relationship_type: String, + target: String, + external: bool, +} + +#[derive(Debug, Default)] +struct CellBuilder { + reference: String, + cell_type: Option, + style_index: Option, + value: String, + inline: String, + formula: String, + formula_kind: Option, + formula_reference: Option, + shared_formula_index: Option, + rich_text: bool, + has_value: bool, + has_inline: bool, + has_formula: bool, + capture_value: bool, + capture_formula: bool, + capture_inline_text: bool, +} + +#[derive(Debug, Default)] +struct SheetScan { + dimension: Option, + semantic: Option, + merged_ranges: Vec, + hidden_rows: usize, + hidden_columns: usize, + cell_elements: usize, + non_empty_cells: usize, + style_only_cells: usize, + features: WorksheetFeatureInventory, +} + +struct ArchiveReader { + archive: ZipArchive, + expanded_bytes: usize, + max_expanded_bytes: usize, +} + +struct CountingReader { + inner: R, + bytes_read: usize, +} + +impl Read for CountingReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buffer)?; + self.bytes_read = self.bytes_read.saturating_add(read); + Ok(read) + } +} + +impl ArchiveReader { + fn new(reader: R, max_expanded_bytes: usize) -> Result { + let archive = ZipArchive::new(reader).map_err(invalid_zip)?; + Ok(Self { + archive, + expanded_bytes: 0, + max_expanded_bytes, + }) + } + + fn entry_count(&self) -> usize { + self.archive.len() + } + + fn feature_inventory(&self) -> WorkbookFeatureInventory { + let mut inventory = WorkbookFeatureInventory::default(); + for name in self.archive.file_names() { + let name = name.to_ascii_lowercase(); + if name.starts_with("xl/theme/") && name.ends_with(".xml") { + inventory.theme_parts += 1; + } else if name.starts_with("xl/drawings/drawing") && name.ends_with(".xml") { + inventory.drawing_parts += 1; + } else if name.starts_with("xl/charts/chart") && name.ends_with(".xml") { + inventory.chart_parts += 1; + } else if name.starts_with("xl/media/") && !name.ends_with('/') { + inventory.image_parts += 1; + } else if name.starts_with("xl/tables/table") && name.ends_with(".xml") { + inventory.table_parts += 1; + } else if name.starts_with("xl/comments") && name.ends_with(".xml") { + inventory.comment_parts += 1; + } else if name.starts_with("xl/ctrlprops/") && name.ends_with(".xml") { + inventory.control_property_parts += 1; + } else if name == "xl/vbaproject.bin" { + inventory.macro_project_parts += 1; + } + } + inventory + } + + fn revision(&mut self) -> Result { + let mut parts = Vec::with_capacity(self.archive.len()); + for index in 0..self.archive.len() { + let entry = self.archive.by_index_raw(index).map_err(invalid_zip)?; + if entry.is_dir() { + continue; + } + if entry.enclosed_name().is_none() { + return Err(ReadError::InvalidXlsx( + "ZIP package contains an unsafe part path".to_owned(), + )); + } + parts.push(WorkbookPartRevision { + name: entry.name().to_owned(), + crc32: format!("{:08x}", entry.crc32()), + compressed_bytes: entry.compressed_size(), + expanded_bytes: entry.size(), + }); + } + parts.sort_by(|left, right| left.name.cmp(&right.name)); + let mut fingerprint = 0xcbf2_9ce4_8422_2325_u64; + let mut compressed_bytes = 0_u64; + let mut expanded_bytes = 0_u64; + for part in &parts { + fnv1a_update(&mut fingerprint, part.name.as_bytes()); + fnv1a_update(&mut fingerprint, part.crc32.as_bytes()); + fnv1a_update(&mut fingerprint, &part.expanded_bytes.to_le_bytes()); + compressed_bytes = compressed_bytes.saturating_add(part.compressed_bytes); + expanded_bytes = expanded_bytes.saturating_add(part.expanded_bytes); + } + Ok(WorkbookRevision { + id: format!("fnv1a64-{fingerprint:016x}"), + compressed_bytes, + expanded_bytes, + parts, + }) + } + + fn read_required_xml(&mut self, name: &str) -> Result { + self.read_xml(name)?.ok_or_else(|| { + ReadError::InvalidXlsx(format!("required OOXML part `{name}` is missing")) + }) + } + + fn read_xml(&mut self, name: &str) -> Result, ReadError> { + let mut entry = match self.archive.by_name(name) { + Ok(entry) => entry, + Err(ZipError::FileNotFound) => return Ok(None), + Err(error) => return Err(invalid_zip(error)), + }; + if entry.is_dir() { + return Err(ReadError::InvalidXlsx(format!( + "OOXML part `{name}` is a directory" + ))); + } + let remaining = self + .max_expanded_bytes + .checked_sub(self.expanded_bytes) + .ok_or(ReadError::SpreadsheetExpandedTooLarge { + limit: self.max_expanded_bytes, + })?; + if entry.size() > u64::try_from(remaining).unwrap_or(u64::MAX) { + return Err(ReadError::SpreadsheetExpandedTooLarge { + limit: self.max_expanded_bytes, + }); + } + let limit = u64::try_from(remaining) + .unwrap_or(u64::MAX) + .saturating_add(1); + let mut bytes = Vec::with_capacity( + usize::try_from(entry.size()) + .unwrap_or(remaining) + .min(remaining), + ); + entry + .by_ref() + .take(limit) + .read_to_end(&mut bytes) + .map_err(|_| { + ReadError::InvalidXlsx(format!("OOXML part `{name}` could not be read")) + })?; + if bytes.len() > remaining { + return Err(ReadError::SpreadsheetExpandedTooLarge { + limit: self.max_expanded_bytes, + }); + } + self.expanded_bytes += bytes.len(); + String::from_utf8(bytes) + .map(Some) + .map_err(|_| ReadError::InvalidXlsx(format!("OOXML part `{name}` is not UTF-8 XML"))) + } + + #[allow(clippy::too_many_arguments)] + fn scan_worksheet( + &mut self, + name: &str, + plans: &[SelectionPlan], + selections: &mut [WorkbookSelection], + shared_strings: &[SharedString], + styles: &Styles, + date_system: DateSystem, + options: &SpreadsheetReadOptions, + relationships: &HashMap, + remaining_cells: &mut usize, + warnings: &mut Vec, + ) -> Result { + let entry = match self.archive.by_name(name) { + Ok(entry) => entry, + Err(ZipError::FileNotFound) => { + return Err(ReadError::InvalidXlsx(format!( + "required OOXML part `{name}` is missing" + ))); + } + Err(error) => return Err(invalid_zip(error)), + }; + if entry.is_dir() { + return Err(ReadError::InvalidXlsx(format!( + "OOXML part `{name}` is a directory" + ))); + } + let remaining = self + .max_expanded_bytes + .checked_sub(self.expanded_bytes) + .ok_or(ReadError::SpreadsheetExpandedTooLarge { + limit: self.max_expanded_bytes, + })?; + let limit = u64::try_from(remaining) + .unwrap_or(u64::MAX) + .saturating_add(1); + let mut counted = CountingReader { + inner: entry.take(limit), + bytes_read: 0, + }; + let result = parse_worksheet( + BufReader::new(&mut counted), + plans, + selections, + shared_strings, + styles, + date_system, + options, + relationships, + remaining_cells, + warnings, + ); + let bytes_read = counted.bytes_read; + self.expanded_bytes = self + .expanded_bytes + .saturating_add(bytes_read.min(remaining)); + if bytes_read > remaining { + return Err(ReadError::SpreadsheetExpandedTooLarge { + limit: self.max_expanded_bytes, + }); + } + result + } +} + +/// Read only ZIP central-directory revision metadata. No OOXML part is +/// expanded, so this is suitable for frequent change probes. +pub fn inspect_workbook_revision( + path: &Path, + max_bytes: usize, +) -> Result { + open_workbook_package(path, max_bytes, usize::MAX).map(|(_package, revision)| revision) +} + +fn open_workbook_package( + path: &Path, + max_bytes: usize, + max_expanded_bytes: usize, +) -> Result<(ArchiveReader, WorkbookRevision), ReadError> { + WorkbookFileStamp::read(path, max_bytes)?; + let file = File::open(path).map_err(|source| ReadError::ReadFile { + path: path.to_path_buf(), + source, + })?; + let mut package = ArchiveReader::new(file, max_expanded_bytes)?; + let revision = package.revision()?; + Ok((package, revision)) +} + +fn apply_revision(result: &mut WorkbookReadResult, revision: WorkbookRevision) { + let part_revisions: HashMap<&str, &WorkbookPartRevision> = revision + .parts + .iter() + .map(|part| (part.name.as_str(), part)) + .collect(); + for sheet in &mut result.workbook.sheets { + if let Some(part) = part_revisions.get(sheet.part.as_str()) { + sheet.revision = (*part).clone(); + } + } + result.revision = revision; +} + +#[allow(clippy::too_many_arguments)] +fn read_xlsx_incremental( + mut package: ArchiveReader, + path: &Path, + options: &ReadOptions, + result: &mut WorkbookReadResult, + context: &WorkbookSessionContext, + revision: WorkbookRevision, + changed_worksheet_parts: &[String], +) -> Result<(), ReadError> { + struct SheetPatch { + sheet_index: usize, + scan: SheetScan, + selections: Vec<(usize, WorkbookSelection)>, + } + + let started = Instant::now(); + let archive_entries = package.entry_count(); + + let mut statistics = WorkbookStatistics { + archive_entries, + ..WorkbookStatistics::default() + }; + let mut warnings = result.warnings.clone(); + let partial_warning = "incremental worksheet refresh is bounded to the cached selections; semantic bounds and worksheet feature inventory are partial"; + if !warnings.iter().any(|warning| warning == partial_warning) { + warnings.push(partial_warning.to_owned()); + } + let changed_sheet_names = changed_worksheet_parts + .iter() + .map(|part| { + result + .workbook + .sheets + .iter() + .find(|sheet| sheet.part == *part) + .map(|sheet| sheet.name.clone()) + .ok_or_else(|| { + ReadError::InvalidXlsx(format!( + "changed worksheet part `{part}` is absent from the cached workbook manifest" + )) + }) + }) + .collect::, _>>()?; + let retained_cells = result + .workbook + .selections + .iter() + .filter(|selection| !changed_sheet_names.contains(&selection.sheet)) + .map(|selection| selection.cells.len()) + .sum::(); + let mut remaining_cells = options.spreadsheet.max_cells.saturating_sub(retained_cells); + let mut patches = Vec::new(); + for part in changed_worksheet_parts { + let sheet_index = result + .workbook + .sheets + .iter() + .position(|sheet| sheet.part == *part) + .ok_or_else(|| { + ReadError::InvalidXlsx(format!( + "changed worksheet part `{part}` is absent from the cached workbook manifest" + )) + })?; + let sheet_name = result.workbook.sheets[sheet_index].name.clone(); + let selected = result + .workbook + .selections + .iter() + .enumerate() + .filter(|(_, selection)| selection.sheet == sheet_name) + .map(|(global_index, selection)| { + let mut selection = selection.clone(); + selection.cells.clear(); + selection.truncated = false; + (global_index, selection) + }) + .collect::>(); + let (global_indexes, mut local_selections): (Vec<_>, Vec<_>) = selected.into_iter().unzip(); + let plans = local_selections + .iter() + .enumerate() + .map(|(output_index, selection)| SelectionPlan { + output_index, + sheet_index, + bounds: selection.bounds, + }) + .collect::>(); + if plans.is_empty() { + return Err(ReadError::InvalidXlsx(format!( + "changed worksheet `{sheet_name}` has no cached selection" + ))); + } + let relationship_path = worksheet_relationship_path(part)?; + let relationships = package + .read_xml(&relationship_path)? + .map(|xml| parse_part_relationships(&xml)) + .transpose()? + .unwrap_or_default(); + let scan = package.scan_worksheet( + part, + &plans, + &mut local_selections, + &context.shared_strings, + &context.styles, + result.workbook.date_system, + &options.spreadsheet, + &relationships, + &mut remaining_cells, + &mut warnings, + )?; + statistics.scanned_sheets += 1; + statistics.cell_elements += scan.cell_elements; + statistics.non_empty_cells += scan.non_empty_cells; + statistics.style_only_cells += scan.style_only_cells; + statistics.returned_cells += local_selections + .iter() + .map(|selection| selection.cells.len()) + .sum::(); + patches.push(SheetPatch { + sheet_index, + scan, + selections: global_indexes.into_iter().zip(local_selections).collect(), + }); + } + statistics.expanded_bytes_read = package.expanded_bytes; + + let mut features = package.feature_inventory(); + features.rich_string_items = context + .shared_strings + .iter() + .filter(|item| item.rich) + .count(); + features.cell_formats = context.styles.cell_formats.len(); + features.custom_number_formats = context.styles.custom_formats.len(); + let mut refreshed_selection_indexes = Vec::new(); + for patch in patches { + for (global_index, selection) in patch.selections { + result.workbook.selections[global_index] = selection; + refreshed_selection_indexes.push(global_index); + } + let sheet = &mut result.workbook.sheets[patch.sheet_index]; + sheet.declared_dimension = patch.scan.dimension; + sheet.semantic_bounds = patch.scan.semantic.map(format_bounds); + sheet.semantic_bounds_complete = patch.scan.features.complete; + sheet.merged_ranges = patch.scan.merged_ranges; + sheet.hidden_rows = patch.scan.hidden_rows; + sheet.hidden_columns = patch.scan.hidden_columns; + sheet.features = patch.scan.features; + } + statistics.returned_cells = result + .workbook + .selections + .iter() + .map(|selection| selection.cells.len()) + .sum(); + if result + .workbook + .selections + .iter() + .any(|selection| selection.truncated) + { + let warning = format!( + "spreadsheet output was truncated at {} returned cells", + options.spreadsheet.max_cells + ); + if !warnings.iter().any(|existing| existing == &warning) { + warnings.push(warning); + } + } + features.inventory_complete = result + .workbook + .sheets + .iter() + .all(|sheet| sheet.features.complete); + result.workbook.features = features; + result.workbook.statistics = statistics; + result.source.bytes = usize::try_from( + std::fs::metadata(path) + .map_err(|source| ReadError::ReadFile { + path: path.to_path_buf(), + source, + })? + .len(), + ) + .unwrap_or(usize::MAX); + result.warnings = warnings; + apply_revision(result, revision); + refreshed_selection_indexes.sort_unstable(); + refreshed_selection_indexes.dedup(); + let markdown_update = + render_markdown_generated_update(&result.workbook, &refreshed_selection_indexes); + let html_update = render_html_generated_update(&result.workbook, &refreshed_selection_indexes); + result.content = merge_generated_blocks(&result.content, &markdown_update)?; + result.content_html = merge_generated_blocks(&result.content_html, &html_update)?; + let elapsed = started.elapsed(); + result.extraction.duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX); + result.extraction.duration_micros = elapsed_micros(elapsed); + Ok(()) +} + +fn elapsed_micros(duration: std::time::Duration) -> u64 { + u64::try_from(duration.as_micros()).unwrap_or(u64::MAX) +} + +pub(crate) fn is_xlsx_path(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("xlsx")) +} + +pub(crate) fn read_xlsx( + path: PathBuf, + options: &ReadOptions, +) -> Result { + read_xlsx_with_context(path, options).map(|(result, _context)| result) +} + +fn read_xlsx_with_context( + path: PathBuf, + options: &ReadOptions, +) -> Result<(WorkbookReadResult, WorkbookSessionContext), ReadError> { + if options.spreadsheet.max_cells == 0 + || options.spreadsheet.max_expanded_bytes == 0 + || options.spreadsheet.preview_rows == 0 + || options.spreadsheet.preview_columns == 0 + { + return Err(ReadError::InvalidXlsx( + "spreadsheet limits must be greater than zero".to_owned(), + )); + } + + let metadata = std::fs::metadata(&path).map_err(|source| ReadError::ReadFile { + path: path.clone(), + source, + })?; + if !metadata.is_file() { + return Err(ReadError::NotRegularFile { path }); + } + if metadata.len() > options.max_bytes as u64 { + return Err(ReadError::InputTooLarge { + limit: options.max_bytes, + }); + } + let canonical = std::fs::canonicalize(&path).map_err(|source| ReadError::ResolveFile { + path: path.clone(), + source, + })?; + let file = File::open(&path).map_err(|source| ReadError::ReadFile { + path: path.clone(), + source, + })?; + + let started = Instant::now(); + let mut package = ArchiveReader::new(file, options.spreadsheet.max_expanded_bytes)?; + let archive_entries = package.entry_count(); + let mut features = package.feature_inventory(); + let revision = package.revision()?; + let part_revisions: HashMap<&str, &WorkbookPartRevision> = revision + .parts + .iter() + .map(|part| (part.name.as_str(), part)) + .collect(); + let content_types = package.read_required_xml("[Content_Types].xml")?; + if !content_types.contains(XLSX_CONTENT_TYPE) { + return Err(ReadError::InvalidXlsx( + "package does not declare an XLSX workbook content type".to_owned(), + )); + } + let workbook_xml = package.read_required_xml("xl/workbook.xml")?; + let relationships_xml = package.read_required_xml("xl/_rels/workbook.xml.rels")?; + let relationships = parse_relationships(&relationships_xml)?; + let mut workbook = parse_workbook(&workbook_xml, &relationships)?; + if workbook.sheets.is_empty() { + return Err(ReadError::InvalidXlsx( + "workbook contains no worksheets".to_owned(), + )); + } + let (shared_strings, styles) = if options.spreadsheet.revision_only { + (Vec::new(), Styles::default()) + } else { + let shared_strings = package + .read_xml("xl/sharedStrings.xml")? + .map(|xml| parse_shared_strings(&xml)) + .transpose()? + .unwrap_or_default(); + let styles = package + .read_xml("xl/styles.xml")? + .map(|xml| parse_styles(&xml)) + .transpose()? + .unwrap_or_default(); + (shared_strings, styles) + }; + features.rich_string_items = shared_strings.iter().filter(|item| item.rich).count(); + features.cell_formats = styles.cell_formats.len(); + features.custom_number_formats = styles.custom_formats.len(); + + let (plans, mut selections) = if options.spreadsheet.revision_only { + (Vec::new(), Vec::new()) + } else { + plan_selections(&workbook.sheets, &options.spreadsheet)? + }; + if let Some(sheet) = workbook + .sheets + .iter() + .find(|sheet| !part_revisions.contains_key(sheet.path.as_str())) + { + return Err(ReadError::InvalidXlsx(format!( + "worksheet part `{}` is missing", + sheet.path + ))); + } + let mut sheets: Vec = workbook + .sheets + .iter() + .enumerate() + .map(|(index, sheet)| WorkbookSheet { + index, + name: sheet.name.clone(), + part: sheet.path.clone(), + revision: (*part_revisions + .get(sheet.path.as_str()) + .expect("worksheet part revisions were validated")) + .clone(), + state: sheet.state, + declared_dimension: None, + semantic_bounds: None, + semantic_bounds_complete: false, + selected: plans.iter().any(|plan| plan.sheet_index == index), + merged_ranges: Vec::new(), + hidden_rows: 0, + hidden_columns: 0, + features: WorksheetFeatureInventory::default(), + }) + .collect(); + let mut statistics = WorkbookStatistics { + archive_entries, + ..WorkbookStatistics::default() + }; + let mut warnings = Vec::new(); + let mut remaining_cells = options.spreadsheet.max_cells; + + let mut plans_by_sheet: BTreeMap> = BTreeMap::new(); + for plan in plans { + plans_by_sheet + .entry(plan.sheet_index) + .or_default() + .push(plan); + } + let scan_all_sheets = + options.spreadsheet.ranges.is_empty() && !options.spreadsheet.revision_only; + let sheet_indexes: Vec = if scan_all_sheets { + (0..workbook.sheets.len()).collect() + } else { + plans_by_sheet.keys().copied().collect() + }; + for sheet_index in sheet_indexes { + let sheet_plans = plans_by_sheet.remove(&sheet_index).unwrap_or_default(); + let descriptor = &workbook.sheets[sheet_index]; + let relationship_path = worksheet_relationship_path(&descriptor.path)?; + let sheet_relationships = package + .read_xml(&relationship_path)? + .map(|xml| parse_part_relationships(&xml)) + .transpose()? + .unwrap_or_default(); + let scan = package.scan_worksheet( + &descriptor.path, + &sheet_plans, + &mut selections, + &shared_strings, + &styles, + workbook.date_system, + &options.spreadsheet, + &sheet_relationships, + &mut remaining_cells, + &mut warnings, + )?; + let sheet = &mut sheets[sheet_index]; + sheet.declared_dimension = scan.dimension; + sheet.semantic_bounds = scan.semantic.map(format_bounds); + sheet.semantic_bounds_complete = scan.features.complete; + sheet.merged_ranges = scan.merged_ranges; + sheet.hidden_rows = scan.hidden_rows; + sheet.hidden_columns = scan.hidden_columns; + sheet.features = scan.features; + statistics.scanned_sheets += 1; + statistics.cell_elements += scan.cell_elements; + statistics.non_empty_cells += scan.non_empty_cells; + statistics.style_only_cells += scan.style_only_cells; + } + statistics.returned_cells = selections + .iter() + .map(|selection| selection.cells.len()) + .sum(); + statistics.expanded_bytes_read = package.expanded_bytes; + features.inventory_complete = statistics.scanned_sheets == workbook.sheets.len() + && sheets.iter().all(|sheet| sheet.features.complete); + if selections.iter().any(|selection| selection.truncated) { + warnings.push(format!( + "spreadsheet output was truncated at {} returned cells", + options.spreadsheet.max_cells + )); + } + if sheets + .iter() + .any(|sheet| sheet.features.scanned && !sheet.features.complete) + { + warnings.push( + "targeted worksheet scan stopped after the requested rows; semantic bounds and worksheet feature inventory are partial" + .to_owned(), + ); + } + if workbook + .defined_names + .iter() + .any(|defined_name| !defined_name.valid_reference) + { + warnings.push("workbook contains defined names with invalid #REF! references".to_owned()); + } + + let title = path + .file_name() + .and_then(|name| name.to_str()) + .map(str::to_owned) + .unwrap_or_else(|| "workbook.xlsx".to_owned()); + let workbook_info = WorkbookInfo { + format: "xlsx", + date_system: workbook.date_system, + sheets, + defined_names: std::mem::take(&mut workbook.defined_names), + selections, + features, + statistics, + }; + let content = render_markdown(&title, &workbook_info); + let content_html = render_html(&title, &workbook_info); + let elapsed = started.elapsed(); + let duration_ms = u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX); + let duration_micros = u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX); + let source = SourceInfo { + kind: SourceKind::File, + requested: path.display().to_string(), + resolved_url: Url::from_file_path(canonical).ok(), + content_type: Some(XLSX_CONTENT_TYPE.to_owned()), + charset: "binary".to_owned(), + bytes: usize::try_from(metadata.len()).unwrap_or(usize::MAX), + }; + + let result = WorkbookReadResult { + schema_version: 1, + artifact_kind: ArtifactKind::Workbook, + content, + content_html, + metadata: WorkbookMetadata { title }, + source, + extraction: WorkbookExtractionInfo { + method: WorkbookExtractionMethod::OoxmlSparse, + duration_ms, + duration_micros, + }, + revision, + workbook: workbook_info, + warnings, + }; + Ok(( + result, + WorkbookSessionContext { + shared_strings, + styles, + }, + )) +} + +fn parse_relationships(xml: &str) -> Result, ReadError> { + let mut reader = xml_reader(xml); + let mut relationships = HashMap::new(); + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"Relationship" => + { + let id = attribute(&element, b"Id")?.ok_or_else(|| { + ReadError::InvalidXlsx("workbook relationship has no Id".to_owned()) + })?; + let target = attribute(&element, b"Target")?.ok_or_else(|| { + ReadError::InvalidXlsx("workbook relationship has no Target".to_owned()) + })?; + let relationship_type = attribute(&element, b"Type")?.unwrap_or_default(); + if relationship_type.ends_with("/worksheet") { + relationships.insert(id, normalize_workbook_target(&target)?); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(relationships) +} + +fn parse_part_relationships(xml: &str) -> Result, ReadError> { + let mut reader = xml_reader(xml); + let mut relationships = HashMap::new(); + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"Relationship" => + { + let id = attribute(&element, b"Id")?.ok_or_else(|| { + ReadError::InvalidXlsx("part relationship has no Id".to_owned()) + })?; + let relationship_type = attribute(&element, b"Type")?.unwrap_or_default(); + let target = attribute(&element, b"Target")?.ok_or_else(|| { + ReadError::InvalidXlsx("part relationship has no Target".to_owned()) + })?; + let external = attribute(&element, b"TargetMode")? + .as_deref() + .is_some_and(|mode| mode.eq_ignore_ascii_case("external")); + relationships.insert( + id, + PartRelationship { + relationship_type, + target, + external, + }, + ); + } + Event::Eof => break, + _ => {} + } + } + Ok(relationships) +} + +fn parse_workbook( + xml: &str, + relationships: &HashMap, +) -> Result { + let mut reader = xml_reader(xml); + let mut date_system = DateSystem::Excel1900; + let mut sheets = Vec::new(); + let mut defined_names = Vec::new(); + let mut current_defined_name: Option<(String, Option, String)> = None; + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"workbookPr" => + { + if attribute(&element, b"date1904")? + .as_deref() + .is_some_and(xml_truthy) + { + date_system = DateSystem::Excel1904; + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"sheet" => + { + let name = attribute(&element, b"name")?.ok_or_else(|| { + ReadError::InvalidXlsx("worksheet entry has no name".to_owned()) + })?; + let relationship_id = attribute_exact(&element, b"r:id")?.ok_or_else(|| { + ReadError::InvalidXlsx(format!("worksheet `{name}` has no relationship id")) + })?; + let path = relationships + .get(&relationship_id) + .cloned() + .ok_or_else(|| { + ReadError::InvalidXlsx(format!( + "worksheet `{name}` relationship target is missing" + )) + })?; + let state = match attribute(&element, b"state")?.as_deref() { + Some("hidden") => SheetState::Hidden, + Some("veryHidden") => SheetState::VeryHidden, + _ => SheetState::Visible, + }; + sheets.push(SheetDescriptor { name, state, path }); + } + Event::Start(element) if local_name(element.name().as_ref()) == b"definedName" => { + let name = attribute(&element, b"name")?.unwrap_or_default(); + let local_sheet_index = + attribute(&element, b"localSheetId")?.and_then(|value| value.parse().ok()); + current_defined_name = Some((name, local_sheet_index, String::new())); + } + Event::Text(text) if current_defined_name.is_some() => { + if let Some((_, _, reference)) = current_defined_name.as_mut() { + reference.push_str(&text.decode().map_err(encoding_error)?); + } + } + Event::GeneralRef(entity) if current_defined_name.is_some() => { + if let Some((_, _, reference)) = current_defined_name.as_mut() { + reference.push_str(&decode_reference(&entity)?); + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"definedName" => { + if let Some((name, local_sheet_index, reference)) = current_defined_name.take() { + defined_names.push(DefinedName { + name, + local_sheet_index, + valid_reference: !reference.contains("#REF!"), + reference, + }); + } + } + Event::Eof => break, + _ => {} + } + } + Ok(ParsedWorkbook { + date_system, + sheets, + defined_names, + }) +} + +fn parse_shared_strings(xml: &str) -> Result, ReadError> { + let mut reader = xml_reader(xml); + let mut strings = Vec::new(); + let mut current: Option = None; + let mut rich = false; + let mut inside_text = false; + let mut phonetic_depth = 0usize; + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) if local_name(element.name().as_ref()) == b"si" => { + current = Some(String::new()); + rich = false; + } + Event::Start(element) if local_name(element.name().as_ref()) == b"r" => { + if current.is_some() { + rich = true; + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"rPh" => { + phonetic_depth += 1; + } + Event::Start(element) + if local_name(element.name().as_ref()) == b"t" && phonetic_depth == 0 => + { + inside_text = true; + } + Event::Text(text) if inside_text => { + if let Some(value) = current.as_mut() { + push_bounded(value, &text.decode().map_err(encoding_error)?); + } + } + Event::GeneralRef(entity) if inside_text => { + if let Some(value) = current.as_mut() { + push_bounded(value, &decode_reference(&entity)?); + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"t" => { + inside_text = false; + } + Event::End(element) if local_name(element.name().as_ref()) == b"rPh" => { + phonetic_depth = phonetic_depth.saturating_sub(1); + } + Event::End(element) if local_name(element.name().as_ref()) == b"si" => { + strings.push(SharedString { + text: current.take().unwrap_or_default(), + rich, + }); + } + Event::Eof => break, + _ => {} + } + } + Ok(strings) +} + +fn parse_styles(xml: &str) -> Result { + let mut reader = xml_reader(xml); + let mut styles = Styles::default(); + let mut inside_cell_xfs = false; + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"numFmt" => + { + if let (Some(id), Some(code)) = ( + attribute(&element, b"numFmtId")?.and_then(|value| value.parse().ok()), + attribute(&element, b"formatCode")?, + ) { + styles.custom_formats.insert(id, code); + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"cellXfs" => { + inside_cell_xfs = true; + } + Event::End(element) if local_name(element.name().as_ref()) == b"cellXfs" => { + inside_cell_xfs = false; + } + Event::Start(element) | Event::Empty(element) + if inside_cell_xfs && local_name(element.name().as_ref()) == b"xf" => + { + let id = attribute(&element, b"numFmtId")? + .and_then(|value| value.parse().ok()) + .unwrap_or(0); + styles.cell_formats.push(id); + } + Event::Eof => break, + _ => {} + } + } + Ok(styles) +} + +#[allow(clippy::too_many_arguments)] +fn parse_worksheet( + input: R, + plans: &[SelectionPlan], + selections: &mut [WorkbookSelection], + shared_strings: &[SharedString], + styles: &Styles, + date_system: DateSystem, + options: &SpreadsheetReadOptions, + relationships: &HashMap, + remaining_cells: &mut usize, + warnings: &mut Vec, +) -> Result { + let mut reader = Reader::from_reader(input); + reader.config_mut().trim_text(false); + let mut buffer = Vec::new(); + let mut scan = SheetScan::default(); + scan.features.scanned = true; + scan.features.complete = true; + let mut cell: Option = None; + let allow_early_stop = !options.ranges.is_empty(); + let max_requested_row = plans.iter().map(|plan| plan.bounds.end_row).max(); + let mut declared_end_row = None; + let mut current_row: Option = None; + let mut phonetic_depth = 0usize; + loop { + match reader.read_event_into(&mut buffer).map_err(xml_error)? { + Event::Start(element) if local_name(element.name().as_ref()) == b"c" => { + scan.cell_elements += 1; + cell = Some(CellBuilder { + reference: attribute(&element, b"r")?.unwrap_or_default(), + cell_type: attribute(&element, b"t")?, + style_index: attribute(&element, b"s")?.and_then(|value| value.parse().ok()), + ..CellBuilder::default() + }); + } + Event::Empty(element) if local_name(element.name().as_ref()) == b"c" => { + scan.cell_elements += 1; + scan.style_only_cells += 1; + } + Event::Start(element) if local_name(element.name().as_ref()) == b"v" => { + if let Some(cell) = cell.as_mut() { + cell.capture_value = true; + cell.has_value = true; + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"f" => { + if let Some(cell) = cell.as_mut() { + cell.capture_formula = true; + cell.has_formula = true; + capture_formula_metadata(cell, &element)?; + scan.features.formula_cells += 1; + } + } + Event::Empty(element) if local_name(element.name().as_ref()) == b"f" => { + if let Some(cell) = cell.as_mut() { + cell.has_formula = true; + capture_formula_metadata(cell, &element)?; + scan.features.formula_cells += 1; + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"t" => { + if let Some(cell) = cell.as_mut() + && cell.cell_type.as_deref() == Some("inlineStr") + { + cell.capture_inline_text = true; + cell.has_inline = true; + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"r" => { + if let Some(cell) = cell.as_mut() + && cell.cell_type.as_deref() == Some("inlineStr") + { + cell.rich_text = true; + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"rPh" => { + phonetic_depth += 1; + } + Event::Text(text) if phonetic_depth == 0 => { + let text = text.decode().map_err(encoding_error)?; + if let Some(cell) = cell.as_mut() { + if cell.capture_value { + push_bounded(&mut cell.value, &text); + } else if cell.capture_formula { + push_bounded(&mut cell.formula, &text); + } else if cell.capture_inline_text { + push_bounded(&mut cell.inline, &text); + } + } + } + Event::GeneralRef(entity) if phonetic_depth == 0 => { + let text = decode_reference(&entity)?; + if let Some(cell) = cell.as_mut() { + if cell.capture_value { + push_bounded(&mut cell.value, &text); + } else if cell.capture_formula { + push_bounded(&mut cell.formula, &text); + } else if cell.capture_inline_text { + push_bounded(&mut cell.inline, &text); + } + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"v" => { + if let Some(cell) = cell.as_mut() { + cell.capture_value = false; + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"f" => { + if let Some(cell) = cell.as_mut() { + cell.capture_formula = false; + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"t" => { + if let Some(cell) = cell.as_mut() { + cell.capture_inline_text = false; + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"rPh" => { + phonetic_depth = phonetic_depth.saturating_sub(1); + } + Event::End(element) if local_name(element.name().as_ref()) == b"c" => { + if let Some(builder) = cell.take() { + if let Some(parsed) = finish_cell( + builder, + shared_strings, + styles, + date_system, + options.include_formulas, + warnings, + )? { + scan.non_empty_cells += 1; + update_semantic_bounds(&mut scan.semantic, parsed.row, parsed.column); + for plan in plans { + if contains(plan.bounds, parsed.row, parsed.column) { + let selection = &mut selections[plan.output_index]; + if *remaining_cells > 0 { + selection.cells.push(parsed.clone()); + *remaining_cells -= 1; + } else { + selection.truncated = true; + } + } + } + } else { + scan.style_only_cells += 1; + } + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"dimension" => + { + scan.dimension = attribute(&element, b"ref")?; + declared_end_row = scan.dimension.as_deref().and_then(|dimension| { + dimension.rsplit_once(':').map_or_else( + || parse_cell_reference(dimension).map(|(row, _)| row), + |(_, end)| parse_cell_reference(end).map(|(row, _)| row), + ) + }); + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"mergeCell" => + { + if let Some(reference) = attribute(&element, b"ref")? { + scan.merged_ranges.push(reference); + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"row" => + { + current_row = attribute(&element, b"r")?.and_then(|value| value.parse().ok()); + if allow_early_stop + && let (Some(row), Some(max_row), Some(sheet_end_row)) = + (current_row, max_requested_row, declared_end_row) + && row > max_row + && sheet_end_row > max_row + { + scan.features.complete = false; + break; + } + if attribute(&element, b"hidden")? + .as_deref() + .is_some_and(xml_truthy) + { + scan.hidden_rows += 1; + } + let outline_level = attribute(&element, b"outlineLevel")? + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + if outline_level > 0 { + scan.features.outlined_rows += 1; + scan.features.max_row_outline_level = + scan.features.max_row_outline_level.max(outline_level); + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"col" => + { + if attribute(&element, b"hidden")? + .as_deref() + .is_some_and(xml_truthy) + { + let min = attribute(&element, b"min")? + .and_then(|value| value.parse::().ok()) + .unwrap_or(1); + let max = attribute(&element, b"max")? + .and_then(|value| value.parse::().ok()) + .unwrap_or(min); + scan.hidden_columns += max.saturating_sub(min).saturating_add(1); + } + let min = attribute(&element, b"min")? + .and_then(|value| value.parse::().ok()) + .unwrap_or(1); + let max = attribute(&element, b"max")? + .and_then(|value| value.parse::().ok()) + .unwrap_or(min); + let outline_level = attribute(&element, b"outlineLevel")? + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + if outline_level > 0 { + scan.features.outlined_columns += max.saturating_sub(min).saturating_add(1); + scan.features.max_column_outline_level = + scan.features.max_column_outline_level.max(outline_level); + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"hyperlink" => + { + let relationship = attribute_exact(&element, b"r:id")? + .and_then(|id| relationships.get(&id)) + .filter(|relationship| relationship.relationship_type.ends_with("/hyperlink")); + let hyperlink = WorkbookHyperlink { + reference: attribute(&element, b"ref")?.unwrap_or_default(), + target: relationship.map(|relationship| relationship.target.clone()), + location: attribute(&element, b"location")?, + display: attribute(&element, b"display")?, + external: relationship.is_some_and(|relationship| relationship.external), + }; + push_feature_reference( + &mut scan.features.hyperlinks, + hyperlink, + &mut scan.features.feature_references_truncated, + ); + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"autoFilter" => + { + scan.features.auto_filter = attribute(&element, b"ref")?; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"conditionalFormatting" => + { + if let Some(reference) = attribute(&element, b"sqref")? { + push_feature_reference( + &mut scan.features.conditional_format_ranges, + reference, + &mut scan.features.feature_references_truncated, + ); + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"cfRule" => + { + scan.features.conditional_format_rules += 1; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"dataValidation" => + { + scan.features.data_validation_rules += 1; + if let Some(reference) = attribute(&element, b"sqref")? { + push_feature_reference( + &mut scan.features.data_validation_ranges, + reference, + &mut scan.features.feature_references_truncated, + ); + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"tablePart" => + { + scan.features.table_parts += 1; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"drawing" => + { + scan.features.drawing_parts += 1; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"legacyDrawing" => + { + scan.features.comment_drawing_parts += 1; + } + Event::Start(element) | Event::Empty(element) + if matches!( + local_name(element.name().as_ref()), + b"pageSetup" | b"pageMargins" | b"printOptions" + ) => + { + scan.features.page_setup = true; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"headerFooter" => + { + scan.features.header_footer = true; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"sparkline" => + { + scan.features.sparklines += 1; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"control" => + { + scan.features.controls += 1; + } + Event::End(element) if local_name(element.name().as_ref()) == b"row" => { + if allow_early_stop + && let (Some(row), Some(max_row), Some(sheet_end_row)) = + (current_row, max_requested_row, declared_end_row) + && row >= max_row + && sheet_end_row > max_row + { + scan.features.complete = false; + break; + } + current_row = None; + } + Event::Eof => break, + _ => {} + } + buffer.clear(); + } + Ok(scan) +} + +fn capture_formula_metadata( + cell: &mut CellBuilder, + element: &BytesStart<'_>, +) -> Result<(), ReadError> { + cell.formula_kind = Some(match attribute(element, b"t")?.as_deref() { + None | Some("normal") => FormulaKind::Normal, + Some("shared") => FormulaKind::Shared, + Some("array") => FormulaKind::Array, + Some("dataTable") => FormulaKind::DataTable, + Some(_) => FormulaKind::Other, + }); + cell.formula_reference = attribute(element, b"ref")?; + cell.shared_formula_index = attribute(element, b"si")?.and_then(|value| value.parse().ok()); + Ok(()) +} + +fn push_feature_reference(target: &mut Vec, value: T, truncated: &mut bool) { + if target.len() < MAX_FEATURE_REFERENCES { + target.push(value); + } else { + *truncated = true; + } +} + +fn finish_cell( + cell: CellBuilder, + shared_strings: &[SharedString], + styles: &Styles, + date_system: DateSystem, + include_formulas: bool, + warnings: &mut Vec, +) -> Result, ReadError> { + if !cell.has_value && !cell.has_inline && !cell.has_formula { + return Ok(None); + } + let (row, column) = parse_cell_reference(&cell.reference).ok_or_else(|| { + ReadError::InvalidXlsx("worksheet cell has an invalid or missing reference".to_owned()) + })?; + let number_format = styles.number_format(cell.style_index); + let raw = if cell.has_inline { + cell.inline + } else { + cell.value + }; + let mut rich_text = cell.rich_text; + let (value_type, display) = match cell.cell_type.as_deref() { + Some("s") => { + let index = raw.parse::().map_err(|_| { + ReadError::InvalidXlsx("shared string cell has an invalid index".to_owned()) + })?; + match shared_strings.get(index) { + Some(value) => { + rich_text = value.rich; + (CellValueType::String, value.text.clone()) + } + None => { + warnings.push(format!( + "cell {} refers to missing shared string index {index}", + cell.reference + )); + (CellValueType::String, raw.clone()) + } + } + } + Some("inlineStr" | "str") => (CellValueType::String, raw.clone()), + Some("b") => ( + CellValueType::Boolean, + if raw == "1" { "TRUE" } else { "FALSE" }.to_owned(), + ), + Some("e") => (CellValueType::Error, raw.clone()), + Some("d") => (CellValueType::Date, raw.clone()), + _ if number_format.as_deref().is_some_and(is_date_format) => { + let display = raw + .parse::() + .ok() + .and_then(|serial| { + format_excel_serial(serial, date_system, number_format.as_deref()) + }) + .unwrap_or_else(|| raw.clone()); + (CellValueType::Date, display) + } + _ if raw.is_empty() => (CellValueType::Blank, String::new()), + _ => (CellValueType::Number, raw.clone()), + }; + let formula = (include_formulas && cell.has_formula && !cell.formula.is_empty()).then(|| { + if cell.formula.starts_with('=') { + cell.formula + } else { + format!("={}", cell.formula) + } + }); + let formula_kind = (include_formulas && cell.has_formula) + .then_some(cell.formula_kind.unwrap_or(FormulaKind::Normal)); + Ok(Some(WorkbookCell { + reference: cell.reference, + row, + column, + value_type, + value: raw, + display, + formula, + formula_kind, + formula_reference: include_formulas.then_some(cell.formula_reference).flatten(), + shared_formula_index: include_formulas + .then_some(cell.shared_formula_index) + .flatten(), + rich_text, + style_index: cell.style_index, + number_format, + })) +} + +fn plan_selections( + sheets: &[SheetDescriptor], + options: &SpreadsheetReadOptions, +) -> Result<(Vec, Vec), ReadError> { + let mut plans = Vec::new(); + let mut selections = Vec::new(); + if options.ranges.is_empty() { + let mut indexes: Vec = sheets + .iter() + .enumerate() + .filter_map(|(index, sheet)| (sheet.state == SheetState::Visible).then_some(index)) + .collect(); + if indexes.is_empty() { + indexes.push(0); + } + let bounds = SelectionBounds { + start_row: 1, + start_column: 1, + end_row: options.preview_rows, + end_column: options.preview_columns, + }; + for sheet_index in indexes { + let sheet = &sheets[sheet_index]; + let range = format_bounds(bounds); + let requested = format!("{}!{range}", quote_sheet_name(&sheet.name)); + push_selection( + &mut plans, + &mut selections, + sheet_index, + sheet.name.clone(), + requested, + bounds, + ); + } + return Ok((plans, selections)); + } + + for selector in &options.ranges { + let (sheet_name, bounds) = parse_selector(selector)?; + let sheet_index = sheets + .iter() + .position(|sheet| sheet.name == sheet_name) + .or_else(|| { + sheets + .iter() + .position(|sheet| sheet.name.eq_ignore_ascii_case(&sheet_name)) + }) + .ok_or_else(|| ReadError::WorksheetNotFound(sheet_name.clone()))?; + push_selection( + &mut plans, + &mut selections, + sheet_index, + sheets[sheet_index].name.clone(), + selector.clone(), + bounds, + ); + } + Ok((plans, selections)) +} + +fn push_selection( + plans: &mut Vec, + selections: &mut Vec, + sheet_index: usize, + sheet: String, + requested: String, + bounds: SelectionBounds, +) { + let output_index = selections.len(); + selections.push(WorkbookSelection { + requested, + sheet, + range: format_bounds(bounds), + bounds, + cells: Vec::new(), + truncated: false, + }); + plans.push(SelectionPlan { + output_index, + sheet_index, + bounds, + }); +} + +fn parse_selector(selector: &str) -> Result<(String, SelectionBounds), ReadError> { + let (sheet, range) = + split_selector(selector).ok_or_else(|| invalid_range(selector, "expected Sheet!A1:D20"))?; + if sheet.is_empty() { + return Err(invalid_range(selector, "worksheet name is empty")); + } + let (start, end) = range.split_once(':').map_or((range, range), |parts| parts); + let (start_row, start_column) = + parse_cell_reference(start).ok_or_else(|| invalid_range(selector, "invalid start cell"))?; + let (end_row, end_column) = + parse_cell_reference(end).ok_or_else(|| invalid_range(selector, "invalid end cell"))?; + if start_row > end_row || start_column > end_column { + return Err(invalid_range( + selector, + "start cell must not follow end cell", + )); + } + Ok(( + sheet, + SelectionBounds { + start_row, + start_column, + end_row, + end_column, + }, + )) +} + +fn split_selector(selector: &str) -> Option<(String, &str)> { + if let Some(rest) = selector.strip_prefix('\'') { + let bytes = rest.as_bytes(); + let mut index = 0; + let mut sheet = String::new(); + while index < bytes.len() { + if bytes[index] == b'\'' { + if bytes.get(index + 1) == Some(&b'\'') { + sheet.push('\''); + index += 2; + continue; + } + if bytes.get(index + 1) == Some(&b'!') { + return Some((sheet, &rest[index + 2..])); + } + return None; + } + let character = rest[index..].chars().next()?; + sheet.push(character); + index += character.len_utf8(); + } + None + } else { + let (sheet, range) = selector.rsplit_once('!')?; + Some((sheet.to_owned(), range)) + } +} + +fn parse_cell_reference(reference: &str) -> Option<(u32, u16)> { + let reference = reference.replace('$', ""); + let split = reference.find(|character: char| character.is_ascii_digit())?; + let (letters, digits) = reference.split_at(split); + if letters.is_empty() + || digits.is_empty() + || !letters + .chars() + .all(|character| character.is_ascii_alphabetic()) + || !digits.chars().all(|character| character.is_ascii_digit()) + { + return None; + } + let mut column: u32 = 0; + for letter in letters.bytes() { + column = column + .checked_mul(26)? + .checked_add(u32::from(letter.to_ascii_uppercase() - b'A' + 1))?; + } + let row: u32 = digits.parse().ok()?; + if row == 0 || row > 1_048_576 || column == 0 || column > 16_384 { + return None; + } + Some((row, u16::try_from(column).ok()?)) +} + +fn contains(bounds: SelectionBounds, row: u32, column: u16) -> bool { + row >= bounds.start_row + && row <= bounds.end_row + && column >= bounds.start_column + && column <= bounds.end_column +} + +fn update_semantic_bounds(bounds: &mut Option, row: u32, column: u16) { + match bounds { + Some(bounds) => { + bounds.start_row = bounds.start_row.min(row); + bounds.start_column = bounds.start_column.min(column); + bounds.end_row = bounds.end_row.max(row); + bounds.end_column = bounds.end_column.max(column); + } + None => { + *bounds = Some(SelectionBounds { + start_row: row, + start_column: column, + end_row: row, + end_column: column, + }); + } + } +} + +fn format_bounds(bounds: SelectionBounds) -> String { + format!( + "{}{}:{}{}", + column_name(bounds.start_column), + bounds.start_row, + column_name(bounds.end_column), + bounds.end_row + ) +} + +fn column_name(column: u16) -> String { + let mut value = u32::from(column); + let mut bytes = Vec::new(); + while value > 0 { + value -= 1; + bytes.push(b'A' + u8::try_from(value % 26).unwrap_or(0)); + value /= 26; + } + bytes.reverse(); + String::from_utf8(bytes).unwrap_or_default() +} + +fn quote_sheet_name(name: &str) -> String { + if name + .chars() + .all(|character| character.is_ascii_alphanumeric() || character == '_') + { + name.to_owned() + } else { + format!("'{}'", name.replace('\'', "''")) + } +} + +fn render_markdown(title: &str, workbook: &WorkbookInfo) -> String { + let mut output = format!("# Workbook: {}\n\n", markdown_escape(title)); + render_markdown_manifest(&mut output, workbook); + for selection in &workbook.selections { + render_markdown_selection(&mut output, selection); + } + output +} + +fn render_markdown_generated_update( + workbook: &WorkbookInfo, + selection_indexes: &[usize], +) -> String { + let mut output = String::new(); + render_markdown_manifest(&mut output, workbook); + for index in selection_indexes { + render_markdown_selection(&mut output, &workbook.selections[*index]); + } + output +} + +fn render_markdown_manifest(output: &mut String, workbook: &WorkbookInfo) { + output.push_str("\n"); + output.push_str("| # | Sheet | State | Declared dimension | Semantic bounds | Selected |\n"); + output.push_str("|---:|---|---|---|---|---|\n"); + for sheet in &workbook.sheets { + output.push_str(&format!( + "| {} | {} | {:?} | {} | {} | {} |\n", + sheet.index + 1, + markdown_escape(&sheet.name), + sheet.state, + sheet.declared_dimension.as_deref().unwrap_or(""), + sheet.semantic_bounds.as_deref().unwrap_or(""), + if sheet.selected { "yes" } else { "no" } + )); + } + if !workbook.defined_names.is_empty() { + output.push_str("\n## Defined names\n\n"); + output.push_str("| Name | Scope | Reference | Valid |\n|---|---|---|---|\n"); + for name in &workbook.defined_names { + output.push_str(&format!( + "| {} | {} | {} | {} |\n", + markdown_escape(&name.name), + name.local_sheet_index + .map(|index| index.to_string()) + .unwrap_or_else(|| "workbook".to_owned()), + markdown_escape(&name.reference), + if name.valid_reference { "yes" } else { "no" } + )); + } + } + output.push_str("\n"); +} + +fn render_markdown_selection(output: &mut String, selection: &WorkbookSelection) { + let marker = selection_marker(&selection.requested); + output.push_str(&format!( + "\n\n" + )); + output.push_str(&format!( + "\n## {}!{}\n\n", + markdown_escape(&selection.sheet), + selection.range + )); + let rows = u64::from(selection.bounds.end_row - selection.bounds.start_row + 1); + let columns = u64::from(selection.bounds.end_column - selection.bounds.start_column + 1); + if rows.saturating_mul(columns) <= PREVIEW_GRID_MAX_AREA && columns <= 50 { + render_markdown_grid(output, selection); + } else { + output.push_str("| Cell | Value | Type | Formula |\n|---|---|---|---|\n"); + for cell in &selection.cells { + output.push_str(&format!( + "| {} | {} | {:?} | {} |\n", + cell.reference, + markdown_escape(&cell.display), + cell.value_type, + markdown_escape(cell.formula.as_deref().unwrap_or("")) + )); + } + } + if selection.truncated { + output.push_str("\n> Selection output was truncated by the cell limit.\n"); + } + output.push_str(&format!("\n")); +} + +/// Merge generated workbook blocks into an agent-maintained Markdown mirror. +/// Text outside generated blocks is preserved byte-for-byte. A malformed +/// existing mirror is rejected instead of being overwritten. +pub fn merge_markdown_mirror( + existing: &str, + update: &WorkbookReadResult, +) -> Result { + if existing.is_empty() { + return Ok(update.content.clone()); + } + merge_generated_blocks(existing, &update.content) +} + +fn merge_generated_blocks(existing: &str, update: &str) -> Result { + let existing_blocks = parse_generated_blocks(existing)?; + if existing_blocks.is_empty() { + return Err(ReadError::InvalidMarkdownMirror( + "existing Markdown has no Opsail generated blocks".to_owned(), + )); + } + let update_blocks = parse_generated_blocks(update)?; + let mut merged = existing.to_owned(); + let mut replacements = Vec::new(); + let mut additions = Vec::new(); + for (id, update_block) in update_blocks { + let replacement = &update[update_block.start..update_block.end]; + if let Some(existing_block) = existing_blocks.get(&id) { + replacements.push((existing_block.start, existing_block.end, replacement)); + } else { + additions.push(replacement); + } + } + replacements.sort_by_key(|(start, _, _)| std::cmp::Reverse(*start)); + for (start, end, replacement) in replacements { + merged.replace_range(start..end, replacement); + } + for addition in additions { + if !merged.ends_with('\n') { + merged.push('\n'); + } + merged.push('\n'); + merged.push_str(addition); + } + Ok(merged) +} + +#[derive(Debug)] +struct GeneratedBlock { + start: usize, + end: usize, +} + +fn parse_generated_blocks(markdown: &str) -> Result, ReadError> { + const PREFIX: &str = ""; + const END_SUFFIX: &str = ":end -->"; + let mut blocks = BTreeMap::new(); + let mut open: Option<(String, usize)> = None; + let mut offset = 0; + for line in markdown.split_inclusive('\n') { + let marker = line.trim_end_matches(['\r', '\n']); + if let Some(id) = marker + .strip_prefix(PREFIX) + .and_then(|value| value.strip_suffix(START_SUFFIX)) + { + if id.is_empty() || open.is_some() || blocks.contains_key(id) { + return Err(ReadError::InvalidMarkdownMirror( + "generated Markdown blocks are nested, empty, or duplicated".to_owned(), + )); + } + open = Some((id.to_owned(), offset)); + } else if let Some(id) = marker + .strip_prefix(PREFIX) + .and_then(|value| value.strip_suffix(END_SUFFIX)) + { + let Some((open_id, start)) = open.take() else { + return Err(ReadError::InvalidMarkdownMirror( + "generated Markdown block has an unmatched end marker".to_owned(), + )); + }; + if open_id != id { + return Err(ReadError::InvalidMarkdownMirror( + "generated Markdown block markers do not match".to_owned(), + )); + } + blocks.insert( + open_id, + GeneratedBlock { + start, + end: offset + line.len(), + }, + ); + } + offset += line.len(); + } + if open.is_some() { + return Err(ReadError::InvalidMarkdownMirror( + "generated Markdown block has no end marker".to_owned(), + )); + } + Ok(blocks) +} + +fn selection_marker(requested: &str) -> String { + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + fnv1a_update(&mut hash, requested.as_bytes()); + format!("selection-{hash:016x}") +} + +fn render_markdown_grid(output: &mut String, selection: &WorkbookSelection) { + let by_position: HashMap<(u32, u16), &WorkbookCell> = selection + .cells + .iter() + .map(|cell| ((cell.row, cell.column), cell)) + .collect(); + output.push_str("| Row |"); + for column in selection.bounds.start_column..=selection.bounds.end_column { + output.push_str(&format!(" {} |", column_name(column))); + } + output.push_str("\n|---:|"); + for _ in selection.bounds.start_column..=selection.bounds.end_column { + output.push_str("---|"); + } + output.push('\n'); + for row in selection.bounds.start_row..=selection.bounds.end_row { + output.push_str(&format!("| {row} |")); + for column in selection.bounds.start_column..=selection.bounds.end_column { + let display = by_position + .get(&(row, column)) + .map_or("", |cell| cell.display.as_str()); + output.push_str(&format!(" {} |", markdown_escape(display))); + } + output.push('\n'); + } +} + +fn render_html(title: &str, workbook: &WorkbookInfo) -> String { + let mut output = format!( + "

Workbook: {}

\n", + html_escape(title) + ); + render_html_manifest(&mut output, workbook); + for selection in &workbook.selections { + render_html_selection(&mut output, selection); + } + output.push_str("
"); + output +} + +fn render_html_generated_update(workbook: &WorkbookInfo, selection_indexes: &[usize]) -> String { + let mut output = String::new(); + render_html_manifest(&mut output, workbook); + for index in selection_indexes { + render_html_selection(&mut output, &workbook.selections[*index]); + } + output +} + +fn render_html_manifest(output: &mut String, workbook: &WorkbookInfo) { + output.push_str("\n
    "); + for sheet in &workbook.sheets { + output.push_str(&format!( + "
  • {} ({:?})
    • ", + html_escape(&sheet.name), + sheet.state + )); + } + output.push_str("
\n\n"); +} + +fn render_html_selection(output: &mut String, selection: &WorkbookSelection) { + let marker = selection_marker(&selection.requested); + output.push_str(&format!( + "\n

{}!{}

", + html_escape(&selection.sheet), + html_escape(&selection.range) + )); + for cell in &selection.cells { + output.push_str(&format!( + "", + html_escape(&cell.reference), + html_escape(&cell.display), + cell.value_type, + html_escape(cell.formula.as_deref().unwrap_or("")) + )); + } + output.push_str(&format!( + "
CellValueTypeFormula
{}{}{:?}{}
\n\n" + )); +} + +fn markdown_escape(value: &str) -> String { + value + .replace('|', "\\|") + .replace("\r\n", "
") + .replace(['\r', '\n'], "
") +} + +fn html_escape(value: &str) -> String { + value + .replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) + .replace('\'', "'") +} + +fn format_excel_serial( + serial: f64, + date_system: DateSystem, + number_format: Option<&str>, +) -> Option { + if !serial.is_finite() || serial < 0.0 { + return None; + } + let mut whole = serial.floor() as i64; + let mut seconds = ((serial - serial.floor()) * 86_400.0).round() as i64; + if seconds >= 86_400 { + whole += 1; + seconds -= 86_400; + } + let has_date = number_format.is_some_and(format_has_date); + let has_time = number_format.is_some_and(format_has_time); + let elapsed_hours = number_format.is_some_and(format_has_elapsed_hours); + let displayed_seconds = if elapsed_hours { + (serial * 86_400.0).round() as i64 + } else { + seconds + }; + let time = format!( + "{:02}:{:02}:{:02}", + displayed_seconds / 3_600, + (displayed_seconds % 3_600) / 60, + displayed_seconds % 60 + ); + if !has_date { + return Some(time); + } + let date = if matches!(date_system, DateSystem::Excel1900) && whole == 60 { + "1900-02-29".to_owned() + } else { + let base = match date_system { + DateSystem::Excel1900 => days_from_civil(1899, 12, 31), + DateSystem::Excel1904 => days_from_civil(1904, 1, 1), + }; + let adjusted = match date_system { + DateSystem::Excel1900 if whole > 60 => whole - 1, + _ => whole, + }; + let (year, month, day) = civil_from_days(base + adjusted); + format!("{year:04}-{month:02}-{day:02}") + }; + if has_time { + Some(format!("{date} {time}")) + } else { + Some(date) + } +} + +fn is_date_format(format: &str) -> bool { + format_has_date(format) || format_has_time(format) +} + +fn format_has_date(format: &str) -> bool { + let format = normalized_number_format(format); + format.contains('y') || format.contains('d') +} + +fn format_has_time(format: &str) -> bool { + let format = normalized_number_format(format); + format.contains('h') || format.contains('s') || format.contains("[m]") +} + +fn format_has_elapsed_hours(format: &str) -> bool { + let format = normalized_number_format(format); + format.contains("[h]") || format.contains("[hh]") +} + +fn normalized_number_format(format: &str) -> String { + let mut normalized = String::new(); + let mut quoted = false; + let mut escaped = false; + let mut bracketed: Option = None; + for character in format.to_ascii_lowercase().chars() { + if let Some(directive) = bracketed.as_mut() { + if character == ']' { + if matches!(directive.as_str(), "h" | "hh" | "m" | "mm" | "s" | "ss") { + normalized.push('['); + normalized.push_str(directive); + normalized.push(']'); + } + bracketed = None; + } else { + directive.push(character); + } + continue; + } + if escaped { + escaped = false; + continue; + } + match character { + '\\' | '_' | '*' if !quoted => escaped = true, + '"' => quoted = !quoted, + '[' if !quoted => bracketed = Some(String::new()), + _ if !quoted => normalized.push(character), + _ => {} + } + } + normalized +} + +fn builtin_number_format(id: u32) -> Option<&'static str> { + match id { + 14 => Some("mm-dd-yy"), + 15 => Some("d-mmm-yy"), + 16 => Some("d-mmm"), + 17 => Some("mmm-yy"), + 18 => Some("h:mm AM/PM"), + 19 => Some("h:mm:ss AM/PM"), + 20 => Some("h:mm"), + 21 => Some("h:mm:ss"), + 22 => Some("m/d/yy h:mm"), + 45 => Some("mm:ss"), + 46 => Some("[h]:mm:ss"), + 47 => Some("mmss.0"), + _ => None, + } +} + +fn days_from_civil(year: i64, month: u32, day: u32) -> i64 { + let year = year - i64::from(month <= 2); + let era = if year >= 0 { year } else { year - 399 } / 400; + let year_of_era = year - era * 400; + let month = i64::from(month); + let day = i64::from(day); + let day_of_year = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let days = days + 719_468; + let era = if days >= 0 { days } else { days - 146_096 } / 146_097; + let day_of_era = days - era * 146_097; + let year_of_era = + (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let mut year = year_of_era + era * 400; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_prime = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_prime + 2) / 5 + 1; + let month = month_prime + if month_prime < 10 { 3 } else { -9 }; + year += i64::from(month <= 2); + (year, month as u32, day as u32) +} + +fn xml_reader(xml: &str) -> Reader<&[u8]> { + let mut reader = Reader::from_str(xml); + reader.config_mut().trim_text(false); + reader +} + +fn attribute(element: &BytesStart<'_>, name: &[u8]) -> Result, ReadError> { + for attribute in element.attributes().with_checks(false) { + let attribute = attribute.map_err(|_| { + ReadError::InvalidXlsx("OOXML element contains an invalid attribute".to_owned()) + })?; + if local_name(attribute.key.as_ref()) == name { + return attribute + .unescape_value() + .map(|value| Some(value.into_owned())) + .map_err(|_| { + ReadError::InvalidXlsx( + "OOXML element contains an invalid attribute value".to_owned(), + ) + }); + } + } + Ok(None) +} + +fn attribute_exact(element: &BytesStart<'_>, name: &[u8]) -> Result, ReadError> { + for attribute in element.attributes().with_checks(false) { + let attribute = attribute.map_err(|_| { + ReadError::InvalidXlsx("OOXML element contains an invalid attribute".to_owned()) + })?; + if attribute.key.as_ref() == name { + return attribute + .unescape_value() + .map(|value| Some(value.into_owned())) + .map_err(|_| { + ReadError::InvalidXlsx( + "OOXML element contains an invalid attribute value".to_owned(), + ) + }); + } + } + Ok(None) +} + +fn local_name(name: &[u8]) -> &[u8] { + name.rsplit(|byte| *byte == b':').next().unwrap_or(name) +} + +fn normalize_workbook_target(target: &str) -> Result { + let target = target.trim_start_matches('/'); + let candidate = if target.starts_with("xl/") { + target.to_owned() + } else { + format!("xl/{target}") + }; + if candidate + .split('/') + .any(|component| component.is_empty() || matches!(component, "." | "..")) + { + return Err(ReadError::InvalidXlsx( + "worksheet relationship target is unsafe".to_owned(), + )); + } + Ok(candidate) +} + +fn worksheet_relationship_path(path: &str) -> Result { + let (directory, file) = path + .rsplit_once('/') + .ok_or_else(|| ReadError::InvalidXlsx("worksheet part path has no directory".to_owned()))?; + if file.is_empty() || directory.is_empty() { + return Err(ReadError::InvalidXlsx( + "worksheet part path is invalid".to_owned(), + )); + } + Ok(format!("{directory}/_rels/{file}.rels")) +} + +fn is_worksheet_part(name: &str) -> bool { + name.starts_with("xl/worksheets/") && name.ends_with(".xml") && !name.contains("/_rels/") +} + +fn worksheet_part_for_change(name: &str) -> Option { + if is_worksheet_part(name) { + return Some(name.to_owned()); + } + let file = name.strip_prefix("xl/worksheets/_rels/")?; + let worksheet = file.strip_suffix(".rels")?; + let part = format!("xl/worksheets/{worksheet}"); + is_worksheet_part(&part).then_some(part) +} + +fn fnv1a_update(hash: &mut u64, bytes: &[u8]) { + for byte in bytes { + *hash ^= u64::from(*byte); + *hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } +} + +fn xml_truthy(value: &str) -> bool { + matches!(value, "1" | "true" | "TRUE") +} + +fn push_bounded(target: &mut String, value: &str) { + if target.len() >= MAX_CELL_TEXT_BYTES { + return; + } + let remaining = MAX_CELL_TEXT_BYTES - target.len(); + if value.len() <= remaining { + target.push_str(value); + return; + } + let mut boundary = remaining; + while boundary > 0 && !value.is_char_boundary(boundary) { + boundary -= 1; + } + target.push_str(&value[..boundary]); +} + +fn invalid_range(selector: &str, reason: &str) -> ReadError { + ReadError::InvalidSpreadsheetRange { + selector: selector.to_owned(), + reason: reason.to_owned(), + } +} + +fn invalid_zip(error: ZipError) -> ReadError { + ReadError::InvalidXlsx(format!("invalid ZIP package: {error}")) +} + +fn xml_error(error: quick_xml::Error) -> ReadError { + ReadError::InvalidXlsx(format!("invalid OOXML: {error}")) +} + +fn encoding_error(error: quick_xml::encoding::EncodingError) -> ReadError { + ReadError::InvalidXlsx(format!("invalid OOXML text encoding: {error}")) +} + +fn decode_reference(reference: &BytesRef<'_>) -> Result { + if let Some(character) = reference.resolve_char_ref().map_err(xml_error)? { + return Ok(character.to_string()); + } + let name = reference.decode().map_err(encoding_error)?; + quick_xml::escape::resolve_predefined_entity(&name) + .map(str::to_owned) + .ok_or_else(|| ReadError::InvalidXlsx(format!("unrecognized OOXML entity `&{name};`"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_quoted_sheet_ranges() { + let (sheet, bounds) = parse_selector("'API Sheet'!$B$6:AY40").unwrap(); + assert_eq!(sheet, "API Sheet"); + assert_eq!(format_bounds(bounds), "B6:AY40"); + + let (sheet, _) = parse_selector("'Owner''s Sheet'!A1").unwrap(); + assert_eq!(sheet, "Owner's Sheet"); + } + + #[test] + fn rejects_out_of_bounds_references() { + assert!(parse_selector("Sheet1!A1:XFD1048576").is_ok()); + assert!(parse_selector("Sheet1!A1:XFE2").is_err()); + assert!(parse_selector("Sheet1!A0:B2").is_err()); + } + + #[test] + fn formats_excel_dates_without_recalculation() { + assert_eq!( + format_excel_serial(45_000.5, DateSystem::Excel1900, Some("yyyy-mm-dd hh:mm")), + Some("2023-03-15 12:00:00".to_owned()) + ); + assert_eq!( + format_excel_serial(0.5, DateSystem::Excel1904, Some("hh:mm")), + Some("12:00:00".to_owned()) + ); + assert_eq!( + format_excel_serial(1.5, DateSystem::Excel1900, Some("[h]:mm:ss")), + Some("36:00:00".to_owned()) + ); + } + + #[test] + fn strips_non_time_bracket_directives_before_date_detection() { + assert!(!is_date_format("[Red]0.00")); + assert!(!is_date_format("[Blue][<=100]0.00")); + assert!(is_date_format("[h]:mm:ss")); + assert!(is_date_format("[$-409]yyyy-mm-dd")); + } +} diff --git a/crates/opsail-read/tests/xlsx.rs b/crates/opsail-read/tests/xlsx.rs new file mode 100644 index 0000000..594b208 --- /dev/null +++ b/crates/opsail-read/tests/xlsx.rs @@ -0,0 +1,709 @@ +use std::io::Write as _; +use std::path::Path; + +use opsail_read::{ + CellValueType, FormulaKind, ReadArtifact, ReadError, ReadOptions, ReadSource, SheetState, + WorkbookPartRevision, WorkbookRevision, WorkbookSession, merge_markdown_mirror, read_artifact, +}; +use tempfile::tempdir; +use zip::ZipWriter; +use zip::write::SimpleFileOptions; + +const CONTENT_TYPES: &str = r#" + + + + + + + + +"#; + +const WORKBOOK: &str = r#" + + + + + + + + #REF! + +"#; + +const RELATIONSHIPS: &str = r#" + + + + + +"#; + +const SHARED_STRINGS: &str = r#" + + Name + A & B +"#; + +const STYLES: &str = r#" + + + + + + +"#; + +const POSITIVE_SHEET: &str = r#" + + + + + 01 + 421SUM(A2,8)5051 + + Merged + + + + + + + + + + + + + + +"#; + +const SHEET_RELATIONSHIPS: &str = r#" + + + + + + +"#; + +const HIDDEN_SHEET: &str = r#" + + hidden +"#; + +const ADVERSARIAL_SHEET: &str = r#" + + + + semantic + + +"#; + +fn write_fixture(path: &Path, first_sheet: &str) { + write_fixture_parts(path, first_sheet, SHARED_STRINGS); +} + +fn write_fixture_parts(path: &Path, first_sheet: &str, shared_strings: &str) { + write_fixture_parts_with_styles(path, first_sheet, shared_strings, STYLES); +} + +fn write_fixture_parts_with_styles( + path: &Path, + first_sheet: &str, + shared_strings: &str, + styles: &str, +) { + let file = std::fs::File::create(path).unwrap(); + let mut zip = ZipWriter::new(file); + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, content) in [ + ("[Content_Types].xml", CONTENT_TYPES), + ("xl/workbook.xml", WORKBOOK), + ("xl/_rels/workbook.xml.rels", RELATIONSHIPS), + ("xl/sharedStrings.xml", shared_strings), + ("xl/styles.xml", styles), + ("xl/worksheets/sheet1.xml", first_sheet), + ("xl/worksheets/_rels/sheet1.xml.rels", SHEET_RELATIONSHIPS), + ("xl/worksheets/sheet2.xml", HIDDEN_SHEET), + ("xl/theme/theme1.xml", ""), + ("xl/drawings/drawing1.xml", ""), + ("xl/charts/chart1.xml", ""), + ("xl/media/image1.png", "fixture"), + ("xl/tables/table1.xml", ""), + ("xl/comments1.xml", ""), + ("xl/ctrlProps/ctrlProp1.xml", ""), + ] { + zip.start_file(name, options).unwrap(); + zip.write_all(content.as_bytes()).unwrap(); + } + zip.finish().unwrap(); +} + +#[tokio::test] +async fn excludes_phonetic_runs_from_shared_string_display_text() { + let directory = tempdir().unwrap(); + let path = directory.path().join("phonetic.xlsx"); + let shared_strings = SHARED_STRINGS.replace( + "Name", + r#"漢字かんじ"#, + ); + write_fixture_parts(&path, POSITIVE_SHEET, &shared_strings); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1".to_owned()]; + + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!("expected workbook artifact"); + }; + + assert_eq!(result.workbook.selections[0].cells[0].display, "漢字"); +} + +#[tokio::test] +async fn color_directives_do_not_turn_numbers_into_dates() { + let directory = tempdir().unwrap(); + let path = directory.path().join("color-format.xlsx"); + let styles = STYLES.replace("yyyy-mm-dd", "[Red]0.00"); + let sheet = POSITIVE_SHEET.replace( + r#"42"#, + r#"42"#, + ); + write_fixture_parts_with_styles(&path, &sheet, SHARED_STRINGS, &styles); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A2".to_owned()]; + + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!("expected workbook artifact"); + }; + let cell = &result.workbook.selections[0].cells[0]; + + assert!(matches!(cell.value_type, CellValueType::Number)); + assert_eq!(cell.display, "42"); +} + +#[tokio::test] +async fn elapsed_hour_formats_preserve_whole_days() { + let directory = tempdir().unwrap(); + let path = directory.path().join("elapsed-hours.xlsx"); + let styles = STYLES.replace("yyyy-mm-dd", "[h]:mm:ss"); + let sheet = POSITIVE_SHEET.replace( + r#"42"#, + r#"1.5"#, + ); + write_fixture_parts_with_styles(&path, &sheet, SHARED_STRINGS, &styles); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A2".to_owned()]; + + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!("expected workbook artifact"); + }; + + assert_eq!(result.workbook.selections[0].cells[0].display, "36:00:00"); +} + +#[test] +fn revision_ignores_zip_recompression_when_part_content_is_unchanged() { + let part = |compressed_bytes| WorkbookPartRevision { + name: "xl/worksheets/sheet1.xml".to_owned(), + crc32: "1234abcd".to_owned(), + compressed_bytes, + expanded_bytes: 4096, + }; + let previous = WorkbookRevision { + id: "previous".to_owned(), + compressed_bytes: 100, + expanded_bytes: 4096, + parts: vec![part(100)], + }; + let repacked = WorkbookRevision { + id: "repacked".to_owned(), + compressed_bytes: 90, + expanded_bytes: 4096, + parts: vec![part(90)], + }; + + let diff = previous.diff(&repacked); + assert!(diff.unchanged); + assert!(diff.changed_parts.is_empty()); +} + +#[test] +fn revision_maps_changed_worksheet_relationships_back_to_their_sheet() { + let revision = |crc32: &str| WorkbookRevision { + id: crc32.to_owned(), + compressed_bytes: 10, + expanded_bytes: 20, + parts: vec![WorkbookPartRevision { + name: "xl/worksheets/_rels/sheet1.xml.rels".to_owned(), + crc32: crc32.to_owned(), + compressed_bytes: 10, + expanded_bytes: 20, + }], + }; + + let diff = revision("11111111").diff(&revision("22222222")); + assert_eq!(diff.changed_worksheet_parts, ["xl/worksheets/sheet1.xml"]); + assert!(!diff.requires_full_refresh); +} + +fn large_tail_sheet(first_value: u32) -> String { + let mut worksheet = format!( + r#" + + + {first_value}"#, + ); + use std::fmt::Write as _; + for row in 2..=20_000 { + write!( + worksheet, + "" + ) + .unwrap(); + } + worksheet.push_str(""); + worksheet +} + +#[tokio::test] +async fn reads_values_formulas_dates_merges_and_visibility() { + let directory = tempdir().unwrap(); + let path = directory.path().join("positive.xlsx"); + write_fixture(&path, POSITIVE_SHEET); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:D4".to_owned()]; + + let artifact = read_artifact(ReadSource::File(path), &options) + .await + .unwrap(); + let ReadArtifact::Workbook(result) = artifact else { + panic!("expected workbook artifact"); + }; + + assert_eq!(result.workbook.sheets.len(), 2); + assert!(matches!( + result.workbook.sheets[1].state, + SheetState::Hidden + )); + assert_eq!( + result.workbook.sheets[0].semantic_bounds.as_deref(), + Some("A1:D4") + ); + assert_eq!(result.workbook.sheets[0].hidden_rows, 1); + assert_eq!(result.workbook.sheets[0].hidden_columns, 1); + assert_eq!(result.workbook.sheets[0].merged_ranges, ["A4:B4"]); + let selection = &result.workbook.selections[0]; + assert_eq!(selection.cells.len(), 8); + assert_eq!( + selection + .cells + .iter() + .find(|cell| cell.reference == "B1") + .unwrap() + .display, + "A & B" + ); + let formula = selection + .cells + .iter() + .find(|cell| cell.reference == "C2") + .unwrap(); + assert_eq!(formula.value, "50"); + assert_eq!(formula.formula.as_deref(), Some("=SUM(A2,8)")); + assert!(matches!(formula.formula_kind, Some(FormulaKind::Shared))); + assert_eq!(formula.formula_reference.as_deref(), Some("C2:D2")); + assert_eq!(formula.shared_formula_index, Some(0)); + let shared_follower = selection + .cells + .iter() + .find(|cell| cell.reference == "D2") + .unwrap(); + assert_eq!(shared_follower.formula, None); + assert!(matches!( + shared_follower.formula_kind, + Some(FormulaKind::Shared) + )); + assert_eq!(shared_follower.shared_formula_index, Some(0)); + assert!( + selection + .cells + .iter() + .find(|cell| cell.reference == "B1") + .unwrap() + .rich_text + ); + assert_eq!( + selection + .cells + .iter() + .find(|cell| cell.reference == "A3") + .unwrap() + .display, + "2023-03-15" + ); + assert!( + result + .warnings + .iter() + .any(|warning| warning.contains("#REF!")) + ); + assert!(result.content.contains("| Row | A | B | C |")); + let features = &result.workbook.sheets[0].features; + assert_eq!(features.formula_cells, 2); + assert_eq!(features.hyperlinks.len(), 1); + assert_eq!( + features.hyperlinks[0].target.as_deref(), + Some("https://example.com") + ); + assert!(features.hyperlinks[0].external); + assert_eq!(features.auto_filter.as_deref(), Some("A1:D4")); + assert_eq!(features.conditional_format_rules, 1); + assert_eq!(features.conditional_format_ranges, ["A2:D2"]); + assert_eq!(features.data_validation_rules, 1); + assert_eq!(features.data_validation_ranges, ["A2:A3"]); + assert_eq!(features.table_parts, 1); + assert_eq!(features.drawing_parts, 1); + assert_eq!(features.comment_drawing_parts, 1); + assert!(features.page_setup); + assert!(features.header_footer); + assert_eq!(features.outlined_rows, 1); + assert_eq!(features.outlined_columns, 1); + assert_eq!(features.max_row_outline_level, 1); + assert_eq!(features.max_column_outline_level, 2); + assert_eq!(features.sparklines, 1); + assert_eq!(features.controls, 1); + assert_eq!(result.workbook.features.rich_string_items, 1); + assert_eq!(result.workbook.features.cell_formats, 2); + assert_eq!(result.workbook.features.custom_number_formats, 1); + assert_eq!(result.workbook.features.theme_parts, 1); + assert_eq!(result.workbook.features.drawing_parts, 1); + assert_eq!(result.workbook.features.chart_parts, 1); + assert_eq!(result.workbook.features.image_parts, 1); + assert_eq!(result.workbook.features.table_parts, 1); + assert_eq!(result.workbook.features.comment_parts, 1); + assert_eq!(result.workbook.features.control_property_parts, 1); +} + +#[tokio::test] +async fn default_preview_does_not_publish_hidden_sheets_but_inventories_them() { + let directory = tempdir().unwrap(); + let path = directory.path().join("preview.xlsx"); + write_fixture(&path, POSITIVE_SHEET); + + let artifact = read_artifact(ReadSource::File(path), &ReadOptions::default()) + .await + .unwrap(); + let ReadArtifact::Workbook(result) = artifact else { + panic!("expected workbook artifact"); + }; + + assert_eq!(result.workbook.selections.len(), 1); + assert_eq!(result.workbook.selections[0].sheet, "Main"); + assert_eq!(result.workbook.statistics.scanned_sheets, 2); + assert!(result.workbook.features.inventory_complete); + assert!(result.workbook.sheets[1].features.scanned); + assert!(!result.workbook.sheets[1].selected); +} + +#[tokio::test] +async fn revision_probe_detects_one_changed_worksheet_without_expanding_it() { + let directory = tempdir().unwrap(); + let path = directory.path().join("revision.xlsx"); + write_fixture(&path, POSITIVE_SHEET); + let initial = read_artifact(ReadSource::File(path.clone()), &ReadOptions::default()) + .await + .unwrap(); + let ReadArtifact::Workbook(initial) = initial else { + panic!("expected workbook artifact"); + }; + + let modified = POSITIVE_SHEET.replacen("42", "43", 1); + write_fixture(&path, &modified); + let mut options = ReadOptions::default(); + options.spreadsheet.revision_only = true; + let current = read_artifact(ReadSource::File(path), &options) + .await + .unwrap(); + let ReadArtifact::Workbook(current) = current else { + panic!("expected workbook artifact"); + }; + + assert!(current.workbook.selections.is_empty()); + assert_eq!(current.workbook.statistics.scanned_sheets, 0); + assert!(!current.workbook.features.inventory_complete); + assert!( + (current.workbook.statistics.expanded_bytes_read as u64) + < current + .revision + .parts + .iter() + .map(|part| part.expanded_bytes) + .sum::() + ); + let diff = initial.revision.diff(¤t.revision); + assert!(!diff.unchanged); + assert!(!diff.requires_full_refresh); + assert_eq!(diff.changed_worksheet_parts, ["xl/worksheets/sheet1.xml"]); +} + +#[tokio::test] +async fn markdown_mirror_refresh_preserves_agent_authored_regions() { + let directory = tempdir().unwrap(); + let path = directory.path().join("mirror.xlsx"); + write_fixture(&path, POSITIVE_SHEET); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:D4".to_owned()]; + let initial = read_artifact(ReadSource::File(path.clone()), &options) + .await + .unwrap(); + let ReadArtifact::Workbook(initial) = initial else { + panic!("expected workbook artifact"); + }; + let existing = format!( + "{}\n## Agent analysis\n\nThis paragraph must survive XLSX refresh.\n", + initial.content + ); + + let modified = POSITIVE_SHEET.replacen("42", "43", 1); + write_fixture(&path, &modified); + let update = read_artifact(ReadSource::File(path), &options) + .await + .unwrap(); + let ReadArtifact::Workbook(update) = update else { + panic!("expected workbook artifact"); + }; + let merged = merge_markdown_mirror(&existing, &update).unwrap(); + + assert!(merged.contains("This paragraph must survive XLSX refresh.")); + assert!(merged.contains("| 2 | 43 | TRUE | 50 | 51 |")); + assert!(!merged.contains("| 2 | 42 | TRUE | 50 | 51 |")); + assert!(matches!( + merge_markdown_mirror("agent notes without markers", &update), + Err(ReadError::InvalidMarkdownMirror(_)) + )); +} + +#[tokio::test] +async fn does_not_allocate_from_a_declared_xfd_dimension() { + let directory = tempdir().unwrap(); + let path = directory.path().join("adversarial.xlsx"); + write_fixture(&path, ADVERSARIAL_SHEET); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:B2".to_owned()]; + + let artifact = read_artifact(ReadSource::File(path), &options) + .await + .unwrap(); + let ReadArtifact::Workbook(result) = artifact else { + panic!("expected workbook artifact"); + }; + + assert_eq!( + result.workbook.sheets[0].declared_dimension.as_deref(), + Some("A1:XFD1048576") + ); + assert_eq!( + result.workbook.sheets[0].semantic_bounds.as_deref(), + Some("A1:A1") + ); + assert!(!result.workbook.sheets[0].semantic_bounds_complete); + assert!(!result.workbook.sheets[0].features.complete); + assert_eq!(result.workbook.statistics.cell_elements, 1); + assert_eq!(result.workbook.statistics.non_empty_cells, 1); + assert_eq!(result.workbook.statistics.style_only_cells, 0); + assert_eq!(result.workbook.statistics.returned_cells, 1); +} + +#[tokio::test] +async fn targeted_streaming_avoids_at_least_eighty_percent_of_expanded_bytes() { + let directory = tempdir().unwrap(); + let path = directory.path().join("large-tail.xlsx"); + let worksheet = large_tail_sheet(1); + write_fixture(&path, &worksheet); + + let full = read_artifact(ReadSource::File(path.clone()), &ReadOptions::default()) + .await + .unwrap(); + let ReadArtifact::Workbook(full) = full else { + panic!("expected workbook artifact"); + }; + let mut targeted_options = ReadOptions::default(); + targeted_options.spreadsheet.ranges = vec!["Main!A1:A1".to_owned()]; + let targeted = read_artifact(ReadSource::File(path), &targeted_options) + .await + .unwrap(); + let ReadArtifact::Workbook(targeted) = targeted else { + panic!("expected workbook artifact"); + }; + + let full_bytes = full.workbook.statistics.expanded_bytes_read as f64; + let targeted_bytes = targeted.workbook.statistics.expanded_bytes_read as f64; + let saved = 1.0 - targeted_bytes / full_bytes; + assert!(saved >= 0.8, "expanded byte saving was {saved:.3}"); + assert!(!targeted.workbook.sheets[0].features.complete); + assert!(full.workbook.features.inventory_complete); +} + +#[test] +fn session_refreshes_human_xlsx_edits_and_preserves_agent_markdown() { + let directory = tempdir().unwrap(); + let path = directory.path().join("collaboration.xlsx"); + write_fixture(&path, &large_tail_sheet(1)); + let mut session = WorkbookSession::open(path.clone(), ReadOptions::default()).unwrap(); + let cold_bytes = session.result().workbook.statistics.expanded_bytes_read; + let existing = format!( + "{}\n## Agent analysis\n\nKeep this analysis across human edits.\n", + session.result().content + ); + + write_fixture(&path, &large_tail_sheet(2)); + let refresh = session.refresh().unwrap(); + let mirror = merge_markdown_mirror(&existing, refresh.result).unwrap(); + let changed = refresh.result.workbook.selections[0] + .cells + .iter() + .find(|cell| cell.reference == "A1") + .unwrap(); + + assert!(refresh.metrics.changed); + assert!(!refresh.metrics.full_refresh); + assert_eq!(refresh.diff.changed_parts, ["xl/worksheets/sheet1.xml"]); + assert!(refresh.metrics.expanded_bytes_read * 5 <= cold_bytes); + assert_eq!(changed.value, "2"); + assert!(mirror.contains("Keep this analysis across human edits.")); + assert!(mirror.contains("| 1 | 2 |")); + assert!(!mirror.contains("| 1 | 1 |")); + let unchanged = session.refresh().unwrap(); + assert!(!unchanged.metrics.changed); + assert_eq!(unchanged.metrics.expanded_bytes_read, 0); +} + +#[test] +fn incremental_refresh_counts_unchanged_cached_cells_against_the_global_limit() { + let directory = tempdir().unwrap(); + let path = directory.path().join("bounded-refresh.xlsx"); + write_fixture(&path, POSITIVE_SHEET); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:D4".to_owned(), "'Hidden Data'!A1".to_owned()]; + options.spreadsheet.max_cells = 9; + let mut session = WorkbookSession::open(path.clone(), options).unwrap(); + assert_eq!( + session + .result() + .workbook + .selections + .iter() + .map(|selection| selection.cells.len()) + .sum::(), + 9 + ); + + let changed_sheet = POSITIVE_SHEET.replace( + r#""#, + r#""#, + ); + write_fixture(&path, &changed_sheet); + let refresh = session.refresh().unwrap(); + let returned = refresh + .result + .workbook + .selections + .iter() + .map(|selection| selection.cells.len()) + .sum::(); + + assert!(!refresh.metrics.full_refresh); + assert_eq!(returned, 9); + assert_eq!(refresh.result.workbook.statistics.returned_cells, 9); + assert!(refresh.result.workbook.selections[0].truncated); + assert_eq!(refresh.result.workbook.selections[1].cells.len(), 1); +} + +#[test] +fn session_fully_refreshes_shared_string_changes_then_reuses_the_new_context() { + let directory = tempdir().unwrap(); + let path = directory.path().join("shared-string-change.xlsx"); + write_fixture(&path, POSITIVE_SHEET); + let mut session = WorkbookSession::open(path.clone(), ReadOptions::default()).unwrap(); + let existing = format!( + "{}\n\n## Agent analysis\n\nshared-string-note\n", + session.result().content + ); + + let changed_strings = SHARED_STRINGS.replace("Name", "Title"); + write_fixture_parts(&path, POSITIVE_SHEET, &changed_strings); + let refresh = session.refresh().unwrap(); + assert!(refresh.metrics.full_refresh); + assert!(refresh.diff.requires_full_refresh); + assert!( + refresh + .diff + .changed_parts + .iter() + .any(|part| part == "xl/sharedStrings.xml") + ); + assert_eq!( + refresh.result.workbook.selections[0].cells[0].display, + "Title" + ); + let mirror = merge_markdown_mirror(&existing, refresh.result).unwrap(); + assert!(mirror.contains("shared-string-note")); + + let changed_sheet = POSITIVE_SHEET.replace("42", "43"); + write_fixture_parts(&path, &changed_sheet, &changed_strings); + let partial = session.refresh().unwrap(); + assert!(!partial.metrics.full_refresh); + let cells = &partial.result.workbook.selections[0].cells; + assert_eq!( + cells + .iter() + .find(|cell| cell.reference == "A1") + .unwrap() + .display, + "Title" + ); + assert_eq!( + cells + .iter() + .find(|cell| cell.reference == "A2") + .unwrap() + .value, + "43" + ); +} + +#[tokio::test] +async fn rejects_invalid_ranges_and_expanded_size_overruns() { + let directory = tempdir().unwrap(); + let path = directory.path().join("limits.xlsx"); + write_fixture(&path, POSITIVE_SHEET); + + let mut invalid = ReadOptions::default(); + invalid.spreadsheet.ranges = vec!["Main!XFE1:XFE2".to_owned()]; + assert!(matches!( + read_artifact(ReadSource::File(path.clone()), &invalid).await, + Err(ReadError::InvalidSpreadsheetRange { .. }) + )); + + let mut bounded = ReadOptions::default(); + bounded.spreadsheet.max_expanded_bytes = 64; + assert!(matches!( + read_artifact(ReadSource::File(path), &bounded).await, + Err(ReadError::SpreadsheetExpandedTooLarge { limit: 64 }) + )); +} diff --git a/crates/opsail/Cargo.toml b/crates/opsail/Cargo.toml index f8c80ed..51138a2 100644 --- a/crates/opsail/Cargo.toml +++ b/crates/opsail/Cargo.toml @@ -31,6 +31,7 @@ url.workspace = true assert_cmd.workspace = true predicates.workspace = true tempfile.workspace = true +zip.workspace = true [lints] workspace = true diff --git a/crates/opsail/src/machine.rs b/crates/opsail/src/machine.rs index ad961e9..1158754 100644 --- a/crates/opsail/src/machine.rs +++ b/crates/opsail/src/machine.rs @@ -4,8 +4,8 @@ use std::process::ExitCode; use std::time::Duration; use opsail_read::{ - CapturedDocument, CdpSource, CdpWaitUntil, ChromeError, ChromeSource, ReadError, ReadOptions, - ReadResult, ReadSource, read, + CapturedDocument, CdpSource, CdpWaitUntil, ChromeError, ChromeSource, ReadArtifact, ReadError, + ReadOptions, ReadSource, read_artifact, }; use serde::{Deserialize, Serialize}; use tokio::io::AsyncReadExt; @@ -83,6 +83,13 @@ enum MachineSource { struct MachineOptions { timeout_ms: Option, max_bytes: Option, + #[serde(default)] + ranges: Vec, + max_cells: Option, + max_expanded_bytes: Option, + include_formulas: Option, + #[serde(default)] + revision_only: bool, } #[derive(Debug, Serialize)] @@ -98,7 +105,7 @@ struct SuccessResponse { protocol_version: u8, ok: bool, engine: EngineInfo, - result: ReadResult, + result: ReadArtifact, } #[derive(Debug, Serialize)] @@ -176,7 +183,7 @@ pub(crate) async fn run() -> ExitCode { } } -async fn execute() -> Result { +async fn execute() -> Result { let bytes = read_request().await?; let request: MachineRequest = serde_json::from_slice(&bytes).map_err(|_| { MachineFailure::new( @@ -198,7 +205,7 @@ async fn execute() -> Result { } let (input, options) = request.into_read_request()?; - read(input, &options) + read_artifact(input, &options) .await .map_err(MachineFailure::from_read_error) } @@ -248,6 +255,37 @@ impl MachineRequest { } options.max_bytes = max_bytes; } + if self.options.ranges.iter().any(String::is_empty) { + return Err(MachineFailure::invalid_option( + "options.ranges must not contain empty selectors", + )); + } + options.spreadsheet.ranges = self.options.ranges; + if self.options.revision_only && !options.spreadsheet.ranges.is_empty() { + return Err(MachineFailure::invalid_option( + "options.revisionOnly cannot be combined with options.ranges", + )); + } + options.spreadsheet.revision_only = self.options.revision_only; + if let Some(max_cells) = self.options.max_cells { + if max_cells == 0 { + return Err(MachineFailure::invalid_option( + "options.maxCells must be greater than zero", + )); + } + options.spreadsheet.max_cells = max_cells; + } + if let Some(max_expanded_bytes) = self.options.max_expanded_bytes { + if max_expanded_bytes == 0 { + return Err(MachineFailure::invalid_option( + "options.maxExpandedBytes must be greater than zero", + )); + } + options.spreadsheet.max_expanded_bytes = max_expanded_bytes; + } + if let Some(include_formulas) = self.options.include_formulas { + options.spreadsheet.include_formulas = include_formulas; + } let input = match self.source { MachineSource::Url { @@ -432,6 +470,36 @@ impl MachineFailure { FailureStage::Input, "source does not appear to be HTML", ), + ReadError::InvalidXlsx(_) => Self::new( + "invalid-xlsx", + FailureStage::Extract, + "source does not appear to be a valid XLSX workbook", + ), + ReadError::InvalidMarkdownMirror(_) => Self::new( + "invalid-markdown-mirror", + FailureStage::Input, + "Markdown mirror contains invalid or conflicting generated block markers", + ), + ReadError::InvalidSpreadsheetRange { .. } => Self::new( + "invalid-spreadsheet-range", + FailureStage::Input, + "spreadsheet range must use Sheet!A1:D20 form and valid Excel bounds", + ), + ReadError::WorksheetNotFound(_) => Self::new( + "worksheet-not-found", + FailureStage::Input, + "the requested spreadsheet worksheet was not found", + ), + ReadError::SpreadsheetExpandedTooLarge { limit } => Self::new( + "spreadsheet-too-large", + FailureStage::Extract, + format!("spreadsheet OOXML exceeds the {limit} expanded byte limit"), + ), + ReadError::SpreadsheetTask => Self::new( + "spreadsheet-extraction-failed", + FailureStage::Extract, + "spreadsheet extraction task failed", + ), ReadError::UnsupportedContentType(_) => Self::new( "unsupported-content-type", FailureStage::Acquire, diff --git a/crates/opsail/src/main.rs b/crates/opsail/src/main.rs index 14334d3..9f14dab 100644 --- a/crates/opsail/src/main.rs +++ b/crates/opsail/src/main.rs @@ -6,7 +6,9 @@ use std::time::Duration; use clap::{ArgAction, Args, Parser, Subcommand, ValueEnum}; use miette::{IntoDiagnostic, Result, WrapErr, miette}; -use opsail_read::{CdpSource, CdpWaitUntil, ChromeSource, Input, ReadOptions, ReadResult, read}; +use opsail_read::{ + CdpSource, CdpWaitUntil, ChromeSource, Input, ReadArtifact, ReadOptions, read_artifact, +}; use serde_json::Value; use tokio::io::AsyncReadExt; use tracing_subscriber::{EnvFilter, util::SubscriberInitExt}; @@ -16,7 +18,7 @@ mod activity; mod codex; mod machine; -const PROPERTY_NAMES: &str = "content, markdown, contentHtml, html, title, author, description, site, published, modified, image, favicon, language, direction, url, canonicalUrl, domain, wordCount, quality, source, extraction"; +const PROPERTY_NAMES: &str = "content, markdown, contentHtml, html, title, author, description, site, published, modified, image, favicon, language, direction, url, canonicalUrl, domain, wordCount, quality, source, extraction, revision, workbook, sheets, selections, definedNames, features, statistics, metrics"; #[derive(Debug, Parser)] #[command( @@ -38,7 +40,7 @@ struct Cli { #[derive(Debug, Subcommand)] enum Command { - /// Read a URL or HTML input and extract its primary content. + /// Read a URL, HTML input, or XLSX workbook. #[command(visible_alias = "extract")] Read(Box), /// Apply a reversible, target-validated application refit. @@ -79,12 +81,18 @@ struct ReadArgs { "chrome_path", "target_id", "cdp_direct", - "wait_until" + "wait_until", + "ranges", + "max_cells", + "max_expanded_bytes", + "max_output_bytes", + "no_formulas", + "revision_only" ] )] machine: bool, - /// URL, HTML file, or '-' for stdin. Browser modes accept an HTTP(S) URL. + /// URL, HTML/XLSX file, or '-' for HTML stdin. Browser modes accept an HTTP(S) URL. #[arg(value_name = "SOURCE")] source: Option, @@ -97,7 +105,8 @@ struct ReadArgs { "content", "markdown", "contentHtml", "html", "title", "author", "description", "site", "published", "modified", "image", "favicon", "language", "direction", "url", "canonicalUrl", "domain", "wordCount", - "quality", "source", "extraction" + "quality", "source", "extraction", "workbook", "sheets", "selections", + "definedNames", "revision", "features", "statistics", "metrics" ])] property: Option, @@ -150,6 +159,39 @@ struct ReadArgs { #[arg(long, value_name = "BYTES", value_parser = parse_positive_usize)] max_bytes: Option, + /// XLSX range in Sheet!A1:D20 form. Repeat to batch multiple ranges. + #[arg( + long = "range", + value_name = "SHEET!A1:D20", + action = ArgAction::Append, + conflicts_with_all = ["cdp", "launch", "chrome_path", "base_url"] + )] + ranges: Vec, + + /// Maximum number of non-empty XLSX cells returned across all ranges. + #[arg(long, value_name = "COUNT", value_parser = parse_positive_usize)] + max_cells: Option, + + /// Maximum cumulative uncompressed OOXML bytes read from an XLSX package. + #[arg(long, value_name = "BYTES", value_parser = parse_positive_usize)] + max_expanded_bytes: Option, + + /// Maximum serialized workbook bytes written to stdout or an output file. + /// Published cells are truncated in stable order to satisfy the bound. + #[arg(long, value_name = "BYTES", value_parser = parse_positive_usize)] + max_output_bytes: Option, + + /// Omit XLSX formula expressions while preserving cached values. + #[arg(long, conflicts_with_all = ["cdp", "launch", "chrome_path"])] + no_formulas: bool, + + /// Read only the XLSX central-directory revision and workbook manifest. + #[arg( + long, + conflicts_with_all = ["cdp", "launch", "chrome_path", "base_url", "ranges"] + )] + revision_only: bool, + /// User-Agent used for HTTP requests or Chrome CDP navigation. #[arg(long, value_name = "VALUE")] user_agent: Option, @@ -337,6 +379,12 @@ async fn run_read(args: ReadArgs) -> Result<()> { wait_until, timeout, max_bytes, + ranges, + max_cells, + max_expanded_bytes, + max_output_bytes, + no_formulas, + revision_only, user_agent, accept_language, } = args; @@ -355,6 +403,15 @@ async fn run_read(args: ReadArgs) -> Result<()> { if let Some(max_bytes) = max_bytes { options.max_bytes = max_bytes; } + options.spreadsheet.ranges = ranges; + if let Some(max_cells) = max_cells { + options.spreadsheet.max_cells = max_cells; + } + if let Some(max_expanded_bytes) = max_expanded_bytes { + options.spreadsheet.max_expanded_bytes = max_expanded_bytes; + } + options.spreadsheet.include_formulas = !no_formulas; + options.spreadsheet.revision_only = revision_only; options.user_agent = user_agent; if let Some(accept_language) = accept_language { options.accept_language = Some(accept_language); @@ -375,16 +432,15 @@ async fn run_read(args: ReadArgs) -> Result<()> { } (Some(_), true) => unreachable!("clap rejects --cdp with --launch"), }; - let result = read(input, &options) + let mut result = read_artifact(input, &options) .await .into_diagnostic() .wrap_err("failed to read source")?; - for warning in &result.warnings { + let data = render_result_bounded(&mut result, format, property.as_deref(), max_output_bytes)?; + for warning in result.warnings() { tracing::warn!(warning = %warning, "read warning"); } - - let data = render_result(&result, format, property.as_deref())?; write_output(output.as_deref(), &data).await } @@ -497,7 +553,7 @@ async fn read_stdin(max_bytes: usize) -> Result> { } fn render_result( - result: &ReadResult, + result: &ReadArtifact, format: OutputFormat, property: Option<&str>, ) -> Result> { @@ -509,8 +565,8 @@ fn render_result( render_property(value, format)? } None => match format { - OutputFormat::Markdown => result.content.clone(), - OutputFormat::Html => result.content_html.clone(), + OutputFormat::Markdown => result.content().to_owned(), + OutputFormat::Html => result.content_html().to_owned(), OutputFormat::Json => serde_json::to_string_pretty(result) .into_diagnostic() .wrap_err("failed to serialize result")?, @@ -520,6 +576,60 @@ fn render_result( Ok(with_trailing_newline(rendered).into_bytes()) } +fn render_result_bounded( + result: &mut ReadArtifact, + format: OutputFormat, + property: Option<&str>, + max_output_bytes: Option, +) -> Result> { + let mut rendered = render_result(result, format, property)?; + let Some(limit) = max_output_bytes else { + return Ok(rendered); + }; + if rendered.len() <= limit { + return Ok(rendered); + } + if !matches!(result, ReadArtifact::Workbook(_)) { + return Err(miette!( + "rendered output is {} bytes, above --max-output-bytes {limit}", + rendered.len() + )); + } + let warning = + format!("spreadsheet output was truncated to fit the {limit} byte serialized-output limit"); + let mut retained_cells = if let ReadArtifact::Workbook(workbook) = result { + if !workbook + .warnings + .iter() + .any(|existing| existing == &warning) + { + workbook.warnings.push(warning); + } + workbook + .workbook + .selections + .iter() + .map(|selection| selection.cells.len()) + .sum::() + } else { + unreachable!("workbook variant checked above") + }; + while rendered.len() > limit && retained_cells > 0 { + retained_cells /= 2; + if let ReadArtifact::Workbook(workbook) = result { + workbook.truncate_published_cells(retained_cells); + } + rendered = render_result(result, format, property)?; + } + if rendered.len() > limit { + return Err(miette!( + "workbook metadata alone is {} bytes, above --max-output-bytes {limit}", + rendered.len() + )); + } + Ok(rendered) +} + fn render_property(value: Value, format: OutputFormat) -> Result { if matches!(format, OutputFormat::Json) { return serde_json::to_string_pretty(&value) diff --git a/crates/opsail/tests/xlsx_cli.rs b/crates/opsail/tests/xlsx_cli.rs new file mode 100644 index 0000000..a60e8fe --- /dev/null +++ b/crates/opsail/tests/xlsx_cli.rs @@ -0,0 +1,194 @@ +use std::io::Write as _; +use std::path::Path; + +use assert_cmd::Command; +use predicates::prelude::*; +use serde_json::json; +use tempfile::tempdir; +use zip::ZipWriter; +use zip::write::SimpleFileOptions; + +fn write_xlsx(path: &Path) { + write_xlsx_with_sheet( + path, + r#"CLI value"#, + ); +} + +fn write_xlsx_with_sheet(path: &Path, sheet: &str) { + let parts = [ + ( + "[Content_Types].xml", + r#""#, + ), + ( + "xl/workbook.xml", + r#""#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#""#, + ), + ("xl/worksheets/sheet1.xml", sheet), + ]; + let file = std::fs::File::create(path).unwrap(); + let mut archive = ZipWriter::new(file); + for (name, xml) in parts { + archive + .start_file(name, SimpleFileOptions::default()) + .unwrap(); + archive.write_all(xml.as_bytes()).unwrap(); + } + archive.finish().unwrap(); +} + +#[test] +fn read_cli_truncates_cells_to_the_serialized_output_budget() { + let directory = tempdir().unwrap(); + let path = directory.path().join("bounded-output.xlsx"); + let value = "x".repeat(1_024); + let mut sheet = String::from( + r#""#, + ); + for row in 1..=20 { + use std::fmt::Write as _; + write!( + sheet, + r#"{value}"# + ) + .unwrap(); + } + sheet.push_str(""); + write_xlsx_with_sheet(&path, &sheet); + let assert = Command::cargo_bin("opsail") + .unwrap() + .args([ + "read", + path.to_str().unwrap(), + "--range", + "Data!A1:A20", + "--max-output-bytes", + "16384", + "--format", + "json", + ]) + .assert() + .success() + .stderr(predicate::str::contains("serialized-output limit")); + let output = assert.get_output(); + let result: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + + assert!(output.stdout.len() <= 16_384); + assert!( + result["workbook"]["statistics"]["returnedCells"] + .as_u64() + .unwrap() + < 20 + ); + assert_eq!(result["workbook"]["selections"][0]["truncated"], true); +} + +#[test] +fn read_cli_emits_a_sparse_workbook_artifact() { + let directory = tempdir().unwrap(); + let path = directory.path().join("cli.xlsx"); + write_xlsx(&path); + let assert = Command::cargo_bin("opsail") + .unwrap() + .args([ + "read", + path.to_str().unwrap(), + "--range", + "Data!A1:B2", + "--format", + "json", + ]) + .assert() + .success() + .stderr(predicate::str::contains("targeted worksheet scan stopped")); + let result: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); + + assert_eq!(result["artifactKind"], "workbook"); + assert_eq!(result["extraction"]["method"], "ooxml-sparse"); + assert_eq!(result["workbook"]["statistics"]["cellElements"], 1); + assert_eq!(result["workbook"]["statistics"]["returnedCells"], 1); + assert_eq!( + result["workbook"]["sheets"][0]["semanticBoundsComplete"], + false + ); + assert_eq!( + result["workbook"]["selections"][0]["cells"][0]["display"], + "CLI value" + ); +} + +#[test] +fn read_cli_revision_only_avoids_worksheet_expansion() { + let directory = tempdir().unwrap(); + let path = directory.path().join("revision.xlsx"); + write_xlsx(&path); + let assert = Command::cargo_bin("opsail") + .unwrap() + .args([ + "read", + path.to_str().unwrap(), + "--revision-only", + "--format", + "json", + ]) + .assert() + .success() + .stderr(""); + let result: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); + + assert_eq!(result["revision"]["parts"].as_array().unwrap().len(), 4); + assert_eq!(result["workbook"]["statistics"]["scannedSheets"], 0); + assert_eq!( + result["workbook"]["selections"].as_array().unwrap().len(), + 0 + ); + assert_eq!( + result["workbook"]["sheets"][0]["features"]["scanned"], + false + ); +} + +#[test] +fn machine_protocol_accepts_batched_xlsx_ranges() { + let directory = tempdir().unwrap(); + let path = directory.path().join("machine.xlsx"); + write_xlsx(&path); + let request = json!({ + "protocolVersion": 1, + "source": {"kind": "file", "path": path}, + "options": { + "ranges": ["Data!A1:A1", "Data!A1:B2"], + "maxCells": 10, + "maxExpandedBytes": 1048576, + "includeFormulas": false + } + }); + let assert = Command::cargo_bin("opsail") + .unwrap() + .args(["read", "--machine"]) + .write_stdin(request.to_string()) + .assert() + .success() + .stderr(""); + let response: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); + + assert_eq!(response["protocolVersion"], 1); + assert_eq!(response["ok"], true); + assert_eq!(response["result"]["artifactKind"], "workbook"); + assert_eq!( + response["result"]["workbook"]["selections"] + .as_array() + .unwrap() + .len(), + 2 + ); + assert_eq!( + response["result"]["workbook"]["statistics"]["scannedSheets"], + 1 + ); +} diff --git a/packages/node/README.md b/packages/node/README.md index c092600..5ba1a25 100644 --- a/packages/node/README.md +++ b/packages/node/README.md @@ -29,6 +29,42 @@ const result = await read({ }); ``` +Local XLSX files return the `WorkbookReadResult` member of the `ReadArtifact` +union. Batch repeated ranges in one request so native code opens the archive +once and scans each selected worksheet once: + +```js +const result = await read({ + source: { kind: "file", path: "./book.xlsx" }, + options: { + ranges: ["Summary!A1:H30", "'API Sheet'!B6:AY40"], + maxCells: 10_000, + maxExpandedBytes: 64 * 1024 * 1024, + includeFormulas: true, + }, +}); +``` + +With no ranges, Opsail returns a bounded preview of visible sheets. It never +allocates a dense rectangle from `declaredDimension`. Formulas are returned +separately from stored cached values and are not recalculated. + +Use `revisionOnly` for a zero-expansion change probe before reusing a caller's +cached result or Markdown mirror. It cannot be combined with ranges: + +```js +const revision = await read({ + source: { kind: "file", path: "./book.xlsx" }, + options: { revisionOnly: true }, +}); +``` + +The process-local `WorkbookSession` cache and generated-block Markdown merge +are Rust-library APIs. The Node wrapper starts one native request per `read()`; +applications that need the same collaboration loop must retain the previous +result, compare revisions, and preserve text outside +`opsail:xlsx-generated` blocks explicitly. + For a `url` source, direct HTTP acquisition sends `opsail/` by default; WeChat article URLs retain Opsail's browser-compatible automatic HTTP profile. Set `options.userAgent` to override either default explicitly. diff --git a/packages/node/index.d.ts b/packages/node/index.d.ts index ac48d62..9472c4b 100644 --- a/packages/node/index.d.ts +++ b/packages/node/index.d.ts @@ -54,6 +54,16 @@ export interface ReadRequest { /** Native acquisition deadline; extraction and bounded cleanup may run afterward. */ timeoutMs?: number; maxBytes?: number; + // Repeated Sheet!A1:D20 selectors, batched in one XLSX archive pass. + ranges?: string[]; + // Maximum non-empty XLSX cells returned across all selectors. + maxCells?: number; + // Maximum cumulative uncompressed OOXML bytes read from an XLSX package. + maxExpandedBytes?: number; + // Include formula expressions alongside cached values. Defaults to true. + includeFormulas?: boolean; + // Read the XLSX revision and workbook manifest without worksheet expansion. + revisionOnly?: boolean; }; } @@ -97,6 +107,163 @@ export interface ReadResult { warnings: string[]; } +export interface WorkbookReadResult { + schemaVersion: 1; + artifactKind: "workbook"; + content: string; + contentHtml: string; + metadata: { title: string }; + source: { + kind: "file"; + requested: string; + resolvedUrl?: string; + contentType?: string; + charset: "binary"; + bytes: number; + }; + extraction: { + method: "ooxml-sparse"; + durationMs: number; + durationMicros: number; + }; + revision: WorkbookRevision; + workbook: { + format: "xlsx"; + dateSystem: "excel1900" | "excel1904"; + sheets: WorkbookSheet[]; + definedNames: WorkbookDefinedName[]; + selections: WorkbookSelection[]; + features: WorkbookFeatureInventory; + statistics: WorkbookStatistics; + }; + warnings: string[]; +} + +export interface WorkbookSheet { + index: number; + name: string; + part: string; + revision: WorkbookPartRevision; + state: "visible" | "hidden" | "very-hidden"; + declaredDimension?: string; + semanticBounds?: string; + semanticBoundsComplete: boolean; + selected: boolean; + mergedRanges: string[]; + hiddenRows: number; + hiddenColumns: number; + features: WorksheetFeatureInventory; +} + +export interface WorkbookRevision { + id: string; + compressedBytes: number; + expandedBytes: number; + parts: WorkbookPartRevision[]; +} + +export interface WorkbookPartRevision { + name: string; + crc32: string; + compressedBytes: number; + expandedBytes: number; +} + +export interface WorkbookFeatureInventory { + inventoryComplete: boolean; + cellFormats: number; + customNumberFormats: number; + richStringItems: number; + themeParts: number; + drawingParts: number; + chartParts: number; + imageParts: number; + tableParts: number; + commentParts: number; + controlPropertyParts: number; + macroProjectParts: number; +} + +export interface WorkbookHyperlink { + reference: string; + target?: string; + location?: string; + display?: string; + external: boolean; +} + +export interface WorksheetFeatureInventory { + scanned: boolean; + complete: boolean; + featureReferencesTruncated: boolean; + formulaCells: number; + hyperlinks: WorkbookHyperlink[]; + autoFilter?: string; + tableParts: number; + drawingParts: number; + commentDrawingParts: number; + conditionalFormatRules: number; + conditionalFormatRanges: string[]; + dataValidationRules: number; + dataValidationRanges: string[]; + pageSetup: boolean; + headerFooter: boolean; + outlinedRows: number; + outlinedColumns: number; + maxRowOutlineLevel: number; + maxColumnOutlineLevel: number; + sparklines: number; + controls: number; +} + +export interface WorkbookDefinedName { + name: string; + localSheetIndex?: number; + reference: string; + validReference: boolean; +} + +export interface WorkbookSelection { + requested: string; + sheet: string; + range: string; + bounds: { + startRow: number; + startColumn: number; + endRow: number; + endColumn: number; + }; + cells: WorkbookCell[]; + truncated: boolean; +} + +export interface WorkbookCell { + reference: string; + row: number; + column: number; + valueType: "string" | "number" | "boolean" | "error" | "date" | "blank"; + value: string; + display: string; + formula?: string; + formulaKind?: "normal" | "shared" | "array" | "data-table" | "other"; + formulaReference?: string; + sharedFormulaIndex?: number; + richText: boolean; + styleIndex?: number; + numberFormat?: string; +} + +export interface WorkbookStatistics { + archiveEntries: number; + expandedBytesRead: number; + scannedSheets: number; + cellElements: number; + nonEmptyCells: number; + styleOnlyCells: number; + returnedCells: number; +} + +export type ReadArtifact = ReadResult | WorkbookReadResult; export interface CallOptions { signal?: AbortSignal; } @@ -109,7 +276,7 @@ export interface OpsailConfig { } export interface OpsailClient { - read(request: ReadRequest, options?: CallOptions): Promise; + read(request: ReadRequest, options?: CallOptions): Promise; } export class OpsailError extends Error { @@ -123,7 +290,7 @@ export class OpsailError extends Error { export function read( request: ReadRequest, options?: CallOptions, -): Promise; +): Promise; export function createOpsail(config?: OpsailConfig): OpsailClient; diff --git a/packages/node/src/client.js b/packages/node/src/client.js index 17878c2..2539b88 100644 --- a/packages/node/src/client.js +++ b/packages/node/src/client.js @@ -413,7 +413,7 @@ export function parseMachineResponse(stdout, exitCode, signalCode, stderr) { }); } - if (response.ok === true && isReadResult(response.result)) { + if (response.ok === true && isReadArtifact(response.result)) { if (exitCode !== 0 || signalCode !== null) { throw new OpsailError("Opsail success response disagrees with its exit code", { code: "protocol-mismatch", @@ -520,6 +520,28 @@ function isEngineInfo(value) { ); } +function isReadArtifact(value) { + return isReadResult(value) || isWorkbookReadResult(value); +} + +function isSourceInfo(value) { + return ( + isRecord(value) && + SOURCE_KINDS.has(value.kind) && + typeof value.requested === "string" && + OPTIONAL_SOURCE_STRINGS.every((field) => isOptionalString(value[field])) && + typeof value.charset === "string" && + isNonNegativeSafeInteger(value.bytes) + ); +} + +function hasStringWarnings(value) { + return ( + Array.isArray(value) && + value.every((warning) => typeof warning === "string") + ); +} + function isReadResult(value) { return ( isRecord(value) && @@ -531,14 +553,7 @@ function isReadResult(value) { OPTIONAL_METADATA_STRINGS.every((field) => isOptionalString(value.metadata[field]), ) && - isRecord(value.source) && - SOURCE_KINDS.has(value.source.kind) && - typeof value.source.requested === "string" && - OPTIONAL_SOURCE_STRINGS.every((field) => - isOptionalString(value.source[field]), - ) && - typeof value.source.charset === "string" && - isNonNegativeSafeInteger(value.source.bytes) && + isSourceInfo(value.source) && isRecord(value.extraction) && EXTRACTION_METHODS.has(value.extraction.method) && isNonNegativeSafeInteger(value.extraction.durationMs) && @@ -550,8 +565,228 @@ function isReadResult(value) { value.quality.extractionRatio >= 0 && value.quality.extractionRatio <= 1 && typeof value.quality.probablyReadable === "boolean" && - Array.isArray(value.warnings) && - value.warnings.every((warning) => typeof warning === "string") + hasStringWarnings(value.warnings) + ); +} + +function isWorkbookReadResult(value) { + return ( + isRecord(value) && + value.schemaVersion === 1 && + value.artifactKind === "workbook" && + typeof value.content === "string" && + typeof value.contentHtml === "string" && + isRecord(value.metadata) && + typeof value.metadata.title === "string" && + isSourceInfo(value.source) && + isRecord(value.extraction) && + value.extraction.method === "ooxml-sparse" && + isNonNegativeSafeInteger(value.extraction.durationMs) && + isNonNegativeSafeInteger(value.extraction.durationMicros) && + isWorkbookRevision(value.revision) && + isWorkbookInfo(value.workbook) && + hasStringWarnings(value.warnings) + ); +} + +function isWorkbookInfo(value) { + return ( + isRecord(value) && + value.format === "xlsx" && + (value.dateSystem === "excel1900" || value.dateSystem === "excel1904") && + Array.isArray(value.sheets) && + value.sheets.every(isWorkbookSheet) && + Array.isArray(value.definedNames) && + value.definedNames.every(isDefinedName) && + Array.isArray(value.selections) && + value.selections.every(isWorkbookSelection) && + isWorkbookFeatureInventory(value.features) && + isWorkbookStatistics(value.statistics) + ); +} + +function isWorkbookSheet(value) { + return ( + isRecord(value) && + isNonNegativeSafeInteger(value.index) && + typeof value.name === "string" && + typeof value.part === "string" && + isWorkbookPartRevision(value.revision) && + ["visible", "hidden", "very-hidden"].includes(value.state) && + isOptionalString(value.declaredDimension) && + isOptionalString(value.semanticBounds) && + typeof value.semanticBoundsComplete === "boolean" && + typeof value.selected === "boolean" && + Array.isArray(value.mergedRanges) && + value.mergedRanges.every((range) => typeof range === "string") && + isNonNegativeSafeInteger(value.hiddenRows) && + isNonNegativeSafeInteger(value.hiddenColumns) && + isWorksheetFeatureInventory(value.features) + ); +} + +function isDefinedName(value) { + return ( + isRecord(value) && + typeof value.name === "string" && + (value.localSheetIndex === undefined || + isNonNegativeSafeInteger(value.localSheetIndex)) && + typeof value.reference === "string" && + typeof value.validReference === "boolean" + ); +} + +function isWorkbookSelection(value) { + return ( + isRecord(value) && + typeof value.requested === "string" && + typeof value.sheet === "string" && + typeof value.range === "string" && + isSelectionBounds(value.bounds) && + Array.isArray(value.cells) && + value.cells.every(isWorkbookCell) && + typeof value.truncated === "boolean" + ); +} + +function isSelectionBounds(value) { + return ( + isRecord(value) && + Number.isSafeInteger(value.startRow) && + value.startRow > 0 && + Number.isSafeInteger(value.startColumn) && + value.startColumn > 0 && + Number.isSafeInteger(value.endRow) && + value.endRow >= value.startRow && + Number.isSafeInteger(value.endColumn) && + value.endColumn >= value.startColumn + ); +} + +function isWorkbookCell(value) { + return ( + isRecord(value) && + typeof value.reference === "string" && + Number.isSafeInteger(value.row) && + value.row > 0 && + Number.isSafeInteger(value.column) && + value.column > 0 && + ["string", "number", "boolean", "error", "date", "blank"].includes( + value.valueType, + ) && + typeof value.value === "string" && + typeof value.display === "string" && + isOptionalString(value.formula) && + (value.formulaKind === undefined || + ["normal", "shared", "array", "data-table", "other"].includes( + value.formulaKind, + )) && + isOptionalString(value.formulaReference) && + (value.sharedFormulaIndex === undefined || + isNonNegativeSafeInteger(value.sharedFormulaIndex)) && + typeof value.richText === "boolean" && + (value.styleIndex === undefined || + isNonNegativeSafeInteger(value.styleIndex)) && + isOptionalString(value.numberFormat) + ); +} + +function isWorkbookRevision(value) { + return ( + isRecord(value) && + /^fnv1a64-[0-9a-f]{16}$/.test(value.id) && + isNonNegativeSafeInteger(value.compressedBytes) && + isNonNegativeSafeInteger(value.expandedBytes) && + Array.isArray(value.parts) && + value.parts.every(isWorkbookPartRevision) + ); +} + +function isWorkbookPartRevision(value) { + return ( + isRecord(value) && + typeof value.name === "string" && + /^[0-9a-f]{8}$/.test(value.crc32) && + isNonNegativeSafeInteger(value.compressedBytes) && + isNonNegativeSafeInteger(value.expandedBytes) + ); +} + +function isWorkbookFeatureInventory(value) { + return ( + isRecord(value) && + typeof value.inventoryComplete === "boolean" && + [ + "cellFormats", + "customNumberFormats", + "richStringItems", + "themeParts", + "drawingParts", + "chartParts", + "imageParts", + "tableParts", + "commentParts", + "controlPropertyParts", + "macroProjectParts", + ].every((field) => isNonNegativeSafeInteger(value[field])) + ); +} + +function isWorksheetFeatureInventory(value) { + return ( + isRecord(value) && + typeof value.scanned === "boolean" && + typeof value.complete === "boolean" && + typeof value.featureReferencesTruncated === "boolean" && + [ + "formulaCells", + "tableParts", + "drawingParts", + "commentDrawingParts", + "conditionalFormatRules", + "dataValidationRules", + "outlinedRows", + "outlinedColumns", + "maxRowOutlineLevel", + "maxColumnOutlineLevel", + "sparklines", + "controls", + ].every((field) => isNonNegativeSafeInteger(value[field])) && + Array.isArray(value.hyperlinks) && + value.hyperlinks.every(isWorkbookHyperlink) && + isOptionalString(value.autoFilter) && + Array.isArray(value.conditionalFormatRanges) && + value.conditionalFormatRanges.every((range) => typeof range === "string") && + Array.isArray(value.dataValidationRanges) && + value.dataValidationRanges.every((range) => typeof range === "string") && + typeof value.pageSetup === "boolean" && + typeof value.headerFooter === "boolean" + ); +} + +function isWorkbookHyperlink(value) { + return ( + isRecord(value) && + typeof value.reference === "string" && + isOptionalString(value.target) && + isOptionalString(value.location) && + isOptionalString(value.display) && + typeof value.external === "boolean" + ); +} + +function isWorkbookStatistics(value) { + return ( + isRecord(value) && + [ + "archiveEntries", + "expandedBytesRead", + "scannedSheets", + "cellElements", + "nonEmptyCells", + "styleOnlyCells", + "returnedCells", + ].every((field) => isNonNegativeSafeInteger(value[field])) ); } diff --git a/packages/node/test/protocol.test.js b/packages/node/test/protocol.test.js index 0deedec..a99c687 100644 --- a/packages/node/test/protocol.test.js +++ b/packages/node/test/protocol.test.js @@ -40,6 +40,135 @@ function validResult() { }; } +function validWorkbookResult() { + return { + schemaVersion: 1, + artifactKind: "workbook", + content: "# Workbook: example.xlsx", + contentHtml: "
", + metadata: { title: "example.xlsx" }, + source: { + kind: "file", + requested: "example.xlsx", + contentType: + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + charset: "binary", + bytes: 1024, + }, + extraction: { method: "ooxml-sparse", durationMs: 2, durationMicros: 2000 }, + revision: { + id: "fnv1a64-0123456789abcdef", + compressedBytes: 1024, + expandedBytes: 2048, + parts: [ + { + name: "xl/worksheets/sheet1.xml", + crc32: "0123abcd", + compressedBytes: 512, + expandedBytes: 1024, + }, + ], + }, + workbook: { + format: "xlsx", + dateSystem: "excel1900", + sheets: [ + { + index: 0, + name: "Data", + part: "xl/worksheets/sheet1.xml", + revision: { + name: "xl/worksheets/sheet1.xml", + crc32: "0123abcd", + compressedBytes: 512, + expandedBytes: 1024, + }, + state: "visible", + declaredDimension: "A1:XFD99", + semanticBounds: "A1:A1", + semanticBoundsComplete: false, + selected: true, + mergedRanges: [], + hiddenRows: 0, + hiddenColumns: 0, + features: { + scanned: true, + complete: false, + featureReferencesTruncated: false, + formulaCells: 0, + hyperlinks: [], + tableParts: 0, + drawingParts: 0, + commentDrawingParts: 0, + conditionalFormatRules: 0, + conditionalFormatRanges: [], + dataValidationRules: 0, + dataValidationRanges: [], + pageSetup: false, + headerFooter: false, + outlinedRows: 0, + outlinedColumns: 0, + maxRowOutlineLevel: 0, + maxColumnOutlineLevel: 0, + sparklines: 0, + controls: 0, + }, + }, + ], + definedNames: [], + selections: [ + { + requested: "Data!A1:B2", + sheet: "Data", + range: "A1:B2", + bounds: { + startRow: 1, + startColumn: 1, + endRow: 2, + endColumn: 2, + }, + cells: [ + { + reference: "A1", + row: 1, + column: 1, + valueType: "string", + value: "value", + display: "value", + richText: false, + }, + ], + truncated: false, + }, + ], + features: { + inventoryComplete: false, + cellFormats: 1, + customNumberFormats: 0, + richStringItems: 0, + themeParts: 1, + drawingParts: 0, + chartParts: 0, + imageParts: 0, + tableParts: 0, + commentParts: 0, + controlPropertyParts: 0, + macroProjectParts: 0, + }, + statistics: { + archiveEntries: 4, + expandedBytesRead: 2048, + scannedSheets: 1, + cellElements: 2, + nonEmptyCells: 1, + styleOnlyCells: 1, + returnedCells: 1, + }, + }, + warnings: [], + }; +} + function successResponse(result = validResult()) { return encode(machineEnvelope({ ok: true, result })); } @@ -68,6 +197,51 @@ test("machine responses require a versioned ReadResult", () => { ); }); +test("machine responses accept a validated sparse workbook artifact", () => { + const result = validWorkbookResult(); + assert.deepEqual(parseMachineResponse(successResponse(result), 0, null), result); +}); + +test("machine workbook validation rejects inconsistent structured fields", () => { + for (const mutate of [ + (result) => { + result.artifactKind = "document"; + }, + (result) => { + result.extraction.method = "semantic"; + }, + (result) => { + result.workbook.sheets[0].state = "secret"; + }, + (result) => { + result.workbook.selections[0].bounds.endRow = 0; + }, + (result) => { + result.workbook.selections[0].cells[0].valueType = "formula"; + }, + (result) => { + result.workbook.statistics.returnedCells = -1; + }, + (result) => { + result.revision.id = "unstable"; + }, + (result) => { + result.workbook.sheets[0].features.complete = "yes"; + }, + (result) => { + result.workbook.selections[0].cells[0].richText = 1; + }, + ]) { + const result = validWorkbookResult(); + mutate(result); + assert.throws( + () => parseMachineResponse(successResponse(result), 0, null), + (error) => + error instanceof OpsailError && error.code === "invalid-response", + ); + } +}); + test("machine responses require the Opsail engine identity", () => { for (const engine of [ undefined, diff --git a/packages/node/test/skill.test.js b/packages/node/test/skill.test.js index 58daca6..72bbd02 100644 --- a/packages/node/test/skill.test.js +++ b/packages/node/test/skill.test.js @@ -108,6 +108,9 @@ test("bootstrap is independently versioned and runtime matches the public packag ]); assert(opsailMetadata.hermes); assert.match(opsailSkill, /`read` extracts readable content/); + assert.match(opsailSkill, /opsail read '\.\/book\.xlsx'/); + assert.match(opsailSkill, /--range 'Summary!A1:H30'/); + assert.match(opsailSkill, /A1:XFD1048576/); assert.match(opsailSkill, /opsail refit codex doctor/); assert.match(opsailSkill, /opsail refit codex enable usage/); assert.match(opsailSkill, /opsail refit codex enable usage --launch/); diff --git a/scripts/benchmark-xlsx-corpus.mjs b/scripts/benchmark-xlsx-corpus.mjs new file mode 100755 index 0000000..e95d5b4 --- /dev/null +++ b/scripts/benchmark-xlsx-corpus.mjs @@ -0,0 +1,307 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { readdir, stat } from "node:fs/promises"; +import { extname, join, resolve } from "node:path"; +import { performance } from "node:perf_hooks"; + +function usage() { + throw new Error( + "usage: benchmark-xlsx-corpus.mjs [--binary PATH] [--concurrency N] [--max-bytes N] [--max-expanded-bytes N] ROOT", + ); +} + +const args = process.argv.slice(2); +let binary = "opsail"; +let concurrency = 8; +let maxBytes = 64 * 1024 * 1024; +let maxExpandedBytes = 512 * 1024 * 1024; +for (let index = 0; index < args.length; ) { + const flag = args[index]; + if (flag === "--binary") { + binary = args[index + 1] ?? usage(); + args.splice(index, 2); + } else if (flag === "--concurrency") { + concurrency = parseBoundedInteger(args[index + 1], 1, 64, flag); + args.splice(index, 2); + } else if (flag === "--max-bytes") { + maxBytes = parseBoundedInteger(args[index + 1], 1, 1024 ** 3, flag); + args.splice(index, 2); + } else if (flag === "--max-expanded-bytes") { + maxExpandedBytes = parseBoundedInteger( + args[index + 1], + 1, + 4 * 1024 ** 3, + flag, + ); + args.splice(index, 2); + } else { + index += 1; + } +} +if (args.length !== 1) usage(); + +function parseBoundedInteger(value, minimum, maximum, flag) { + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new Error(`${flag} must be an integer from ${minimum} to ${maximum}`); + } + return parsed; +} + +async function findWorkbooks(root) { + const workbooks = []; + const directories = [root]; + while (directories.length > 0) { + const directory = directories.pop(); + const entries = await readdir(directory, { withFileTypes: true }); + entries.sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + const path = join(directory, entry.name); + if (entry.isDirectory()) { + directories.push(path); + } else if ( + entry.isFile() && + !entry.name.startsWith("~$") && + extname(entry.name).toLowerCase() === ".xlsx" + ) { + workbooks.push(path); + } + } + } + workbooks.sort(); + return workbooks; +} + +function runWorkbook(path, { revisionOnly = false, range } = {}) { + return new Promise((resolveResult) => { + const command = [ + "read", + "--format", + "json", + "--property", + "metrics", + "--max-bytes", + String(maxBytes), + "--max-expanded-bytes", + String(maxExpandedBytes), + "--max-cells", + "1", + ]; + if (revisionOnly) command.push("--revision-only"); + if (range !== undefined) command.push("--range", range); + command.push(path); + const started = performance.now(); + const child = spawn(binary, command, { stdio: ["ignore", "pipe", "pipe"] }); + const stdout = []; + const stderr = []; + let outputBytes = 0; + let outputLimited = false; + const capture = (chunks, chunk) => { + outputBytes += chunk.length; + if (outputBytes <= 1024 * 1024) chunks.push(chunk); + else outputLimited = true; + }; + child.stdout.on("data", (chunk) => capture(stdout, chunk)); + child.stderr.on("data", (chunk) => capture(stderr, chunk)); + child.on("error", (error) => { + resolveResult({ + ok: false, + wallMs: performance.now() - started, + errorKind: "spawn", + diagnostic: error.message, + }); + }); + child.on("close", (code, signal) => { + const wallMs = performance.now() - started; + const stdoutText = Buffer.concat(stdout).toString("utf8"); + const stderrText = Buffer.concat(stderr).toString("utf8"); + if (code !== 0 || signal !== null || outputLimited) { + resolveResult({ + ok: false, + wallMs, + errorKind: outputLimited ? "output-limit" : `exit-${code ?? signal}`, + diagnostic: boundedDiagnostic(stderrText), + }); + return; + } + try { + const metrics = JSON.parse(stdoutText); + if ( + metrics.extraction?.method !== "ooxml-sparse" || + !Number.isSafeInteger(metrics.extraction.durationMicros) || + metrics.extraction.durationMicros < 0 || + !Number.isSafeInteger(metrics.statistics?.expandedBytesRead) || + !Array.isArray(metrics.sheets) + ) { + throw new Error("invalid metrics property"); + } + resolveResult({ + ok: true, + wallMs, + metrics, + }); + } catch (error) { + resolveResult({ + ok: false, + wallMs, + errorKind: "invalid-output", + diagnostic: error.message, + }); + } + }); + }); +} + +function boundedDiagnostic(value) { + return value + .replaceAll(/\x1b\[[0-9;]*m/g, "") + .replaceAll(/[\r\n]+/g, " ") + .trim() + .slice(0, 500); +} + +function percentile(values, fraction) { + if (values.length === 0) return null; + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.ceil(fraction * sorted.length) - 1]; +} + +const corpusRoot = resolve(args[0]); +const workbooks = await findWorkbooks(corpusRoot); +if (workbooks.length === 0) throw new Error("no XLSX workbooks found"); +const sizes = await Promise.all(workbooks.map(async (path) => (await stat(path)).size)); +const benchmarkStarted = performance.now(); +let nextIndex = 0; +const results = new Array(workbooks.length); +async function worker() { + while (nextIndex < workbooks.length) { + const index = nextIndex; + nextIndex += 1; + const cold = await runWorkbook(workbooks[index]); + if (!cold.ok) { + results[index] = { ...cold, stage: "cold" }; + continue; + } + const sheet = + cold.metrics.sheets.find(({ state }) => state === "visible") ?? + cold.metrics.sheets[0]; + if (sheet === undefined || typeof sheet.name !== "string") { + results[index] = { + ok: false, + wallMs: cold.wallMs, + errorKind: "no-worksheet", + diagnostic: "workbook metrics contain no worksheet", + stage: "plan", + }; + continue; + } + const revision = await runWorkbook(workbooks[index], { revisionOnly: true }); + if (!revision.ok) { + results[index] = { ...revision, stage: "revision" }; + continue; + } + const range = `${quoteSheetName(sheet.name)}!A1:P24`; + const targeted = await runWorkbook(workbooks[index], { range }); + if (!targeted.ok) { + results[index] = { ...targeted, stage: "targeted" }; + continue; + } + const coldBytes = cold.metrics.statistics.expandedBytesRead; + const incrementalBytes = + revision.metrics.statistics.expandedBytesRead + + targeted.metrics.statistics.expandedBytesRead; + const coldMicros = cold.metrics.extraction.durationMicros; + const incrementalMicros = + revision.metrics.extraction.durationMicros + + targeted.metrics.extraction.durationMicros; + const byteSaving = coldBytes === 0 ? 0 : 1 - incrementalBytes / coldBytes; + const timeSaving = coldMicros === 0 ? 0 : 1 - incrementalMicros / coldMicros; + results[index] = { + ok: true, + wallMs: cold.wallMs + revision.wallMs + targeted.wallMs, + cold, + revision, + targeted, + byteSaving, + timeSaving, + efficient: byteSaving >= 0.8 && timeSaving >= 0.8, + }; + } +} + +function quoteSheetName(name) { + if (/^[A-Za-z0-9_]+$/.test(name)) return name; + return `'${name.replaceAll("'", "''")}'`; +} +await Promise.all(Array.from({ length: Math.min(concurrency, workbooks.length) }, worker)); +const wallMs = performance.now() - benchmarkStarted; +const successes = results.filter((result) => result.ok); +const failures = results + .map((result, index) => ({ result, index })) + .filter(({ result }) => !result.ok); +const failureKinds = {}; +for (const { result } of failures) { + failureKinds[result.errorKind] = (failureKinds[result.errorKind] ?? 0) + 1; +} +const totalBytes = sizes.reduce((sum, value) => sum + value, 0); +const successRatePercent = (successes.length / workbooks.length) * 100; +const thresholdPercent = 80; +const efficient = successes.filter((result) => result.efficient); +const efficiencyRatePercent = (efficient.length / successes.length) * 100; +const output = { + schemaVersion: 1, + benchmark: "opsail-xlsx-corpus", + corpusRoot, + binary, + concurrency, + acceptance: { + compatibility: { + metric: "successful cold, revision, and targeted reads / eligible non-temporary XLSX files", + actualPercent: Number(successRatePercent.toFixed(3)), + }, + highEfficiency: { + metric: "valid workbooks with both expanded-byte and extraction-time savings >= 80%", + thresholdPercent, + actualPercent: Number(efficiencyRatePercent.toFixed(3)), + passed: efficiencyRatePercent >= thresholdPercent, + }, + }, + corpus: { + workbooks: workbooks.length, + successful: successes.length, + failed: failures.length, + compressedBytes: totalBytes, + largestWorkbookBytes: Math.max(...sizes), + }, + performance: { + wallMs: Number(wallMs.toFixed(3)), + throughputMiBPerSecond: Number( + (totalBytes / (1024 * 1024) / (wallMs / 1000)).toFixed(3), + ), + processWallMs: { + p50: Number(percentile(successes.map(({ wallMs }) => wallMs), 0.5)?.toFixed(3)), + p95: Number(percentile(successes.map(({ wallMs }) => wallMs), 0.95)?.toFixed(3)), + p99: Number(percentile(successes.map(({ wallMs }) => wallMs), 0.99)?.toFixed(3)), + max: Number(Math.max(...successes.map(({ wallMs }) => wallMs)).toFixed(3)), + }, + expandedByteSavingPercent: { + p50: Number((percentile(successes.map(({ byteSaving }) => byteSaving), 0.5) * 100).toFixed(3)), + p95: Number((percentile(successes.map(({ byteSaving }) => byteSaving), 0.95) * 100).toFixed(3)), + }, + extractionTimeSavingPercent: { + p50: Number((percentile(successes.map(({ timeSaving }) => timeSaving), 0.5) * 100).toFixed(3)), + p95: Number((percentile(successes.map(({ timeSaving }) => timeSaving), 0.95) * 100).toFixed(3)), + }, + }, + failures: { + byKind: failureKinds, + samples: failures.slice(0, 20).map(({ result, index }) => ({ + path: workbooks[index].slice(corpusRoot.length + 1), + kind: result.errorKind, + stage: result.stage, + diagnostic: result.diagnostic, + })), + }, +}; +process.stdout.write(`${JSON.stringify(output, null, 2)}\n`); diff --git a/scripts/benchmark-xlsx-read.mjs b/scripts/benchmark-xlsx-read.mjs new file mode 100755 index 0000000..655120d --- /dev/null +++ b/scripts/benchmark-xlsx-read.mjs @@ -0,0 +1,163 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { performance } from "node:perf_hooks"; + +function usage() { + throw new Error( + "usage: benchmark-xlsx-read.mjs [--binary PATH] [--runs N] WORKBOOK RANGE [RANGE ...]", + ); +} + +const args = process.argv.slice(2); +let binary = "opsail"; +let runs = 7; +for (let index = 0; index < args.length; ) { + if (args[index] === "--binary") { + binary = args[index + 1] ?? usage(); + args.splice(index, 2); + } else if (args[index] === "--runs") { + runs = Number(args[index + 1]); + if (!Number.isSafeInteger(runs) || runs < 1 || runs > 100) usage(); + args.splice(index, 2); + } else { + index += 1; + } +} +if (args.length < 2) usage(); +const [workbook, ...ranges] = args; + +function invoke(selectedRanges) { + const command = [ + "read", + workbook, + "--format", + "json", + "--max-cells", + "100000", + ]; + for (const range of selectedRanges) command.push("--range", range); + const started = performance.now(); + const result = spawnSync(binary, command, { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + }); + const elapsedMs = performance.now() - started; + if (result.status !== 0 || result.signal !== null) { + throw new Error( + `opsail failed with status ${result.status ?? "signal"}: ${result.stderr.trim()}`, + ); + } + const artifact = JSON.parse(result.stdout); + if (artifact.artifactKind !== "workbook") { + throw new Error("opsail did not return a workbook artifact"); + } + return { + elapsedMs, + outputBytes: Buffer.byteLength(result.stdout), + artifact, + }; +} + +function selectionProjection(artifact) { + return artifact.workbook.selections.map((selection) => ({ + requested: selection.requested, + sheet: selection.sheet, + range: selection.range, + truncated: selection.truncated, + cells: selection.cells.map((cell) => ({ + reference: cell.reference, + valueType: cell.valueType, + value: cell.value, + display: cell.display, + formula: cell.formula, + styleIndex: cell.styleIndex, + numberFormat: cell.numberFormat, + })), + })); +} + +function digest(value) { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +} + +function summarize(samples, projection) { + return { + runs, + medianMs: Number(median(samples.map((sample) => sample.elapsedMs)).toFixed(3)), + minMs: Number(Math.min(...samples.map((sample) => sample.elapsedMs)).toFixed(3)), + maxMs: Number(Math.max(...samples.map((sample) => sample.elapsedMs)).toFixed(3)), + medianOutputBytes: median(samples.map((sample) => sample.outputBytes)), + semanticDigest: digest(projection(samples[0])), + }; +} + +invoke(ranges); +const batchedSamples = Array.from({ length: runs }, () => invoke(ranges)); + +function invokeIsolated() { + const started = performance.now(); + const results = ranges.map((range) => invoke([range])); + return { + elapsedMs: performance.now() - started, + outputBytes: results.reduce((sum, result) => sum + result.outputBytes, 0), + artifacts: results.map((result) => result.artifact), + }; +} + +invokeIsolated(); +const isolatedSamples = Array.from({ length: runs }, invokeIsolated); + +invoke([]); +const previewSamples = Array.from({ length: runs }, () => invoke([])); + +const batched = summarize(batchedSamples, (sample) => + selectionProjection(sample.artifact), +); +const isolated = summarize(isolatedSamples, (sample) => + sample.artifacts.flatMap(selectionProjection), +); +const preview = summarize(previewSamples, (sample) => + selectionProjection(sample.artifact), +); + +if (batched.semanticDigest !== isolated.semanticDigest) { + throw new Error("batched and isolated range projections differ"); +} + +const batchedArtifact = batchedSamples[0].artifact; +process.stdout.write( + `${JSON.stringify( + { + schemaVersion: 1, + workbook, + rangeCount: ranges.length, + uniqueSelectedSheets: + batchedArtifact.workbook.statistics.scannedSheets, + returnedCells: + batchedArtifact.workbook.statistics.returnedCells, + expandedBytesRead: + batchedArtifact.workbook.statistics.expandedBytesRead, + styleOnlyCells: + batchedArtifact.workbook.statistics.styleOnlyCells, + batched, + isolated, + defaultPreview: preview, + speedupVsIsolated: Number( + (isolated.medianMs / batched.medianMs).toFixed(2), + ), + semanticMatch: true, + }, + null, + 2, + )}\n`, +); diff --git a/skills/opsail/SKILL.md b/skills/opsail/SKILL.md index e96ced6..7e71ea7 100644 --- a/skills/opsail/SKILL.md +++ b/skills/opsail/SKILL.md @@ -1,6 +1,6 @@ --- name: opsail -description: Use the Opsail native CLI for reliable agent capabilities, including readable-content extraction and the target-validated Codex sidebar usage refit. +description: Use the Opsail native CLI for readable HTML extraction, sparse local XLSX range reading, and the target-validated Codex sidebar usage refit. license: Apache-2.0 compatibility: Requires Opsail 0.2.0 and terminal execution. metadata: {"author":"Opsail contributors","version":"0.2.0","homepage":"https://github.com/lencx/opsail","openclaw":{"emoji":"⛵","homepage":"https://github.com/lencx/opsail","requires":{"bins":["opsail"]},"install":[{"id":"node","kind":"node","package":"opsail@0.2.0","bins":["opsail"],"label":"Install Opsail (npm)"}]},"hermes":{"tags":["opsail","native-tools","content-extraction","markdown","agents"]}} @@ -8,7 +8,7 @@ metadata: {"author":"Opsail contributors","version":"0.2.0","homepage":"https:// # Opsail -Use the `opsail` native CLI for capabilities exposed through its unified command entry point. `read` extracts readable content from static HTML or a DOM rendered by an isolated or caller-managed Chrome. `refit codex` manages a reversible, target-validated usage display in the Codex sidebar. If the binary is missing or not version `0.2.0`, ask the user to provide and authorize `https://raw.githubusercontent.com/lencx/opsail/refs/heads/main/skills/bootstrap-opsail/SKILL.md`; do not install it implicitly from this Skill. +Use the `opsail` native CLI for capabilities exposed through its unified command entry point. `read` extracts readable content from static HTML or a DOM rendered by an isolated or caller-managed Chrome, and reads bounded local XLSX ranges through a sparse OOXML path. `refit codex` manages a reversible, target-validated usage display in the Codex sidebar. If the binary is missing or not version `0.2.0`, ask the user to provide and authorize `https://raw.githubusercontent.com/lencx/opsail/refs/heads/main/skills/bootstrap-opsail/SKILL.md`; do not install it implicitly from this Skill. ## Read: choose the source @@ -38,6 +38,53 @@ Get-Content -Raw '.\article.html' | opsail read - Prefer an argument array when invoking Opsail from code. Quote shell URLs and paths, and pass untrusted or large HTML through a file or stdin rather than interpolating it into a command. +## Read: select XLSX ranges + +Read a local `.xlsx` workbook as a bounded sparse artifact. Structured JSON preserves sheet +visibility, declared dimensions, semantic bounds, defined names, formulas and cached values, +merged ranges, selected cells, truncation, and scan statistics: + +```sh +opsail read './book.xlsx' --format json +``` + +With no `--range`, Opsail previews a bounded rectangle from each visible sheet. Hidden sheets +remain in the manifest but are not previewed. When the task names one or more regions, repeat +`--range` in a single invocation so the ZIP package, shared strings, and styles are opened once +and each selected worksheet is scanned once: + +```sh +opsail read './book.xlsx' \ + --range 'Summary!A1:H30' \ + --range "'API Sheet'!B6:AY40" \ + --max-cells 10000 \ + --format json +``` + +`--max-expanded-bytes` limits cumulative uncompressed OOXML bytes. `--no-formulas` omits formula +expressions but preserves cached values. Formula values are never recalculated and may be stale. +Treat `declaredDimension`, `semanticBounds`, `styleOnlyCells`, `truncated`, invalid defined names, +and warnings as separate evidence. + +For a repeated read, probe the content revision before rereading cell XML: + +```sh +opsail read './book.xlsx' --revision-only --property revision --format json +``` + +Reuse an existing Markdown mirror when the revision is unchanged. When a worksheet part changed, +reread the previously selected ranges in one invocation and replace only matching +`opsail:xlsx-generated` blocks; keep agent-authored text outside those blocks. Workbook, +shared-string, style, or theme changes are dependency-wide and require a conservative full read. +The CLI invocation itself is stateless; the high-efficiency `WorkbookSession` cache is an +in-process `opsail-read` Rust API, so do not claim session-level timings for repeated CLI processes. + +Positive use means selecting semantic regions and checking `truncated: false`. Do not request +`A1:XFD1048576` or all sheets merely because the declared dimension is large; do not infer that +style-only cells contain data; and do not use Chrome, PDF conversion, or screenshots as the +default XLSX read path. Legacy `.xls`, encrypted workbooks, macro execution, chart reconstruction, +image OCR, and Excel-accurate visual rendering are outside this capability. + ## Read: use Chrome rendering when needed Use owned launch when a page requires rendering and a fresh, unauthenticated browser session is sufficient: From 303cdcb9f51f77ccfaf451e920004912280ec0a1 Mon Sep 17 00:00:00 2001 From: jieyuexing Date: Sat, 29 Aug 2026 15:45:27 +0800 Subject: [PATCH 2/2] feat(read): preserve XLSX layout and visual evidence --- Cargo.lock | 1 + crates/opsail-read/Cargo.toml | 1 + crates/opsail-read/README.md | 16 +- crates/opsail-read/XLSX_COMPATIBILITY.md | 87 +- crates/opsail-read/src/lib.rs | 13 +- crates/opsail-read/src/model.rs | 3 +- crates/opsail-read/src/xlsx.rs | 1917 +++++++++++++++++++++- crates/opsail-read/tests/xlsx.rs | 517 +++++- crates/opsail/tests/xlsx_cli.rs | 104 +- packages/node/index.d.ts | 99 ++ packages/node/src/client.js | 140 ++ packages/node/test/protocol.test.js | 88 +- skills/opsail/SKILL.md | 10 +- 13 files changed, 2871 insertions(+), 125 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb7ad34..03698b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1423,6 +1423,7 @@ dependencies = [ "opsail-chrome", "quick-xml", "reqwest", + "ring", "rustls", "serde", "serde_json", diff --git a/crates/opsail-read/Cargo.toml b/crates/opsail-read/Cargo.toml index d1e3e0a..e38fef4 100644 --- a/crates/opsail-read/Cargo.toml +++ b/crates/opsail-read/Cargo.toml @@ -18,6 +18,7 @@ futures-util.workspace = true opsail-chrome = { path = "../opsail-chrome", version = "0.1.0" } quick-xml.workspace = true reqwest.workspace = true +ring.workspace = true rustls.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/opsail-read/README.md b/crates/opsail-read/README.md index 3718f33..476a8c1 100644 --- a/crates/opsail-read/README.md +++ b/crates/opsail-read/README.md @@ -31,6 +31,9 @@ browser path. - Inventory hyperlinks, filters, tables, conditional formats, validations, page setup, outlines, drawings, charts, images, comments, controls, sparklines, themes, and VBA-project presence without executing active content. +- Resolve worksheet picture anchors and media relationships, inventory their + content type/size/SHA-256, and publish bounded data-URI pixels only for + pictures intersecting selected ranges. - Compare ZIP-part revisions without expanding OOXML, refresh changed worksheet selections through `WorkbookSession`, and merge generated Markdown blocks without overwriting agent-authored text. @@ -101,8 +104,8 @@ such as `A1:XFD1048576` never causes rectangular allocation. With no explicit ranges, visible sheets receive a bounded preview controlled by `preview_rows` and `preview_columns`; hidden sheets remain in the manifest. -`max_cells` bounds published non-empty cells and marks affected selections as -truncated. `max_expanded_bytes` limits cumulative uncompressed OOXML read from +`max_cells` bounds published cell records, including blank merge-covered cells, +and marks affected selections as truncated. `max_expanded_bytes` limits cumulative uncompressed OOXML read from the ZIP package. The normal `max_bytes` limit still applies to the compressed file itself. Stdout adapters can call `WorkbookReadResult::truncate_published_cells` and rerender a stable prefix when their serialized transport has a stricter byte @@ -110,11 +113,12 @@ budget than the cell-count limit. Formula expressions and cached values are reported separately. Opsail does not recalculate formulas, so callers must not describe the cached value as current. -Legacy `.xls`, encrypted workbooks, macros, charts, images, conditional-format -rendering, and Excel-accurate visual layout are outside this reader. +Legacy `.xls`, encrypted workbooks, macros, chart reconstruction, image OCR, +textbox/VML extraction, conditional-format rendering, and Excel-accurate visual +layout are outside this reader. -The read-side feature levels, completeness labels, human/agent mirror contract, -and 80% efficiency gate are specified in +The read-side feature levels, tail/cell-data completeness labels, range-overflow +evidence, and human/agent mirror contract are specified in [`XLSX_COMPATIBILITY.md`](XLSX_COMPATIBILITY.md). ## Refresh a human-owned workbook without losing agent Markdown diff --git a/crates/opsail-read/XLSX_COMPATIBILITY.md b/crates/opsail-read/XLSX_COMPATIBILITY.md index 45bfa99..230c95b 100644 --- a/crates/opsail-read/XLSX_COMPATIBILITY.md +++ b/crates/opsail-read/XLSX_COMPATIBILITY.md @@ -16,21 +16,21 @@ is an Excel renderer. | rust_xlsxwriter persisted feature | Opsail read level | Current evidence | | --- | --- | --- | | Basic Excel data types | Semantic | Strings, numbers, booleans, errors, blanks, and stored dates | -| Cell formatting | Structural | Style index, number format, format counts; fonts/fills/borders are not resolved yet | +| Cell formatting | Semantic for font strike and color; structural otherwise | Style index and number format remain available. Cell font strike and OOXML color (`rgb`, `theme`, `indexed`, `auto`, `tint`) are published; `resolvedRgb` is added only when direct RGB, theme, or the standard indexed palette can be resolved, with tint applied. Fills/borders are not resolved. | | Formulas | Semantic | Expression, cached value, normal/shared/array/data-table kind, shared index and formula range; no recalculation | | Charts | Structural | Package chart and drawing-part counts | | Hyperlinks | Semantic | Cell reference, internal location or relationship target, display text, external flag | -| Page/printing setup | Structural | Page-setup/header-footer presence plus print-area/title defined names | -| Merged ranges | Semantic when the worksheet scan is complete | Exact merge references | +| Page/printing setup | Structural | Per-sheet print-area/title defined names, page setup attributes, print options, bounded row/column break ids, and header/footer presence | +| Merged ranges | Semantic | Exact sheet and intersecting-selection merge references; selected anchor and covered cells publish merge membership without copying the anchor value | | Conditional formatting | Structural | Bounded `sqref` list and rule count | | Data validation | Structural | Bounded `sqref` list and rule count | | Cell notes | Structural | Comment-part and legacy-drawing counts | | Textboxes | Structural | Drawing presence only; textbox text is not extracted | | Checkboxes | Structural | Worksheet control and control-property part counts | | Sparklines | Structural | Sparkline element count | -| Images | Structural | Media and drawing-part counts; pixels and anchors are not rendered | +| Images | Semantic when a bounded payload is returned; structural otherwise | Worksheet `twoCellAnchor`/`oneCellAnchor` pictures publish anchor cells/indexes, media part, MIME type, bytes, and SHA-256. Intersecting selections may add a capped `dataUri`; sheet inventories and cap-overflow entries remain metadata-only. No OCR or layout rendering. | | Workbook themes | Structural | Theme-part count | -| Rich multi-format strings | Semantic text, structural runs | Concatenated base text plus a `richText` flag; East Asian `rPh` pronunciation hints are excluded from cell text and per-run formatting is not published | +| Rich multi-format strings | Semantic text and font formatting | Concatenated `value`/`display` plus the compatible `richText` flag are retained. `richTextRuns[]` publishes each run's text, strike, and OOXML font color, including provable `resolvedRgb`. East Asian `rPh` pronunciation hints remain excluded from cell text. | | Outline groupings | Semantic | Outlined row/column counts and maximum levels | | Defined names | Semantic | Name, scope, reference, and invalid `#REF!` signal | | Autofilters | Semantic | Exact worksheet filter range | @@ -44,11 +44,43 @@ capability rows. ## Completeness labels Default preview scans every worksheet to build a complete workbook feature -inventory, while publishing cells only from visible-sheet preview ranges. -Explicit range reads may stop after the last requested row. They set -`semanticBoundsComplete=false`, `features.complete=false`, and emit a warning -when the tail was not scanned. Declared worksheet dimensions never authorize a -dense allocation. +inventory, while publishing cells only from visible-sheet preview ranges. An +explicit range read may skip the contents of later `sheetData` rows, but it +always advances to worksheet EOF and parses post-`sheetData` merges, +autofilters, page/print setup, header/footer, and row/column breaks. + +`features.cellDataComplete` reports whether every cell body was inspected; +`features.tailFeaturesComplete` reports whether the worksheet tail was reached; +the compatibility `features.complete` flag is true only when both are true. +`semanticBoundsComplete` follows cell-data completeness. Selection `truncated` +still means the cell cap prevented publication, never that worksheet tail +features were unread. Declared worksheet dimensions never authorize a dense +allocation. + +Each selection publishes `usedBounds` from non-empty cells on its requested +rows, plus bounded left/right column overflow and any above/below row overflow +that was actually scanned. `mergedRanges` contains every merge intersecting the +requested rectangle, including merges whose anchor or far edge is outside it. +Returned cells in a merge carry `merge.range`, `merge.anchor`, and +`merge.role`. A requested covered cell may therefore be published as a blank +cell record; the anchor value is not invented on it. + +For each scanned sheet, `pictures[]` inventories DrawingML worksheet pictures +without embedding pixels, bounded to 256 references per sheet/selection. Each entry publishes `sheet`, `fromCell`, optional +`toCell`, the corresponding zero-based row/column marker indexes, `mediaPart`, +`contentType`, `byteSize`, and `sha256`. Selection `images[]` contains only +pictures whose anchor intersects the requested rectangle. It adds `dataUri` +when the image fits the 2 MiB per-image and 4 MiB total raw-payload caps; +otherwise `payloadTruncated` is true and the metadata remains. The selection's +`imagesTruncated` reports image payload/reference limits independently from the +cell-only `truncated` flag. Base64 expansion stays within the Host 16 MiB +serialized-output ceiling. Picture bytes are not OCR text and are never copied +into cells. Charts, textboxes, and VML are not interpreted as pictures. + +Per-sheet `print` evidence contains `printArea`, `printTitles`, structured +`pageSetup` (`orientation`, `paperSize`, fit/scale attributes), optional +`printOptions`, bounded `rowBreaks`/`columnBreaks`, and `headerFooter`. This is a +range hint, not reconstructed pagination or a rendering of margins. ## Human/agent collaboration loop @@ -65,8 +97,8 @@ replaceable generated blocks and unrestricted text outside those blocks. 4. A worksheet-only change uses the already-open package, refreshes the prior selections for changed sheets, reuses unchanged selections, and replaces only the corresponding generated Markdown/HTML blocks. -5. Workbook, shared-string, style, or theme changes trigger a conservative full - refresh. +5. Workbook, shared-string, style, theme, drawing, or media changes trigger a + conservative full refresh. 6. `merge_markdown_mirror` replaces only Opsail generated blocks. It preserves agent-authored text byte-for-byte and rejects malformed/nested markers. @@ -76,14 +108,17 @@ Compatibility and efficiency are separate metrics. - Compatibility rate = successful cold, revision, and targeted reads divided by eligible non-temporary XLSX files. -- Expanded-byte saving = `1 - incremental expanded bytes / cold expanded bytes`. -- Read-time saving = `1 - incremental refresh wall time / cold-read wall time`. -- A collaboration read is **high efficiency** only when both savings are at - least 80% and its refreshed selection matches a cold-read semantic projection. +- A bounded worksheet read limits cell materialization and skips parsing later + cell subtrees, but it still decompresses the worksheet through EOF to retain + tail evidence. It therefore does not promise an expanded-byte saving over a + full worksheet scan. +- Revision probes and unchanged refreshes still expand zero worksheet bytes. +- Read-time and retained-cell savings remain useful metrics, provided the + refreshed selection matches a cold-read semantic projection. - The agent-Markdown gate runs repeated unchanged refreshes across every valid workbook. The direct human-edit gate uses size-stratified real workbook samples, edits one visible numeric non-formula cell in a temporary copy, and - requires at least 80% of samples to meet the high-efficiency definition. The + records bounded-read timing and retained-cell counts. The declared repeated-Markdown plus one-human-edit cycle is reported separately. Concurrent throughput cannot turn a slow individual read into a pass. @@ -92,6 +127,24 @@ intentional negative example: it can affect many cells and therefore must take the full-refresh path. The direct worksheet-only efficiency number must not be generalized to those dependency-wide edits. +## Text-format protocol + +Cells optionally publish `fontStrike`, `fontColor`, and `richTextRuns`. A color +preserves its OOXML source fields (`rgb`, `theme`, `indexed`, `auto`, `tint`). +`resolvedRgb` is a six-digit uppercase sRGB value only when Opsail can prove it +from direct RGB, `theme1.xml`, or the standard 0-63 indexed palette; automatic, +missing, out-of-range, or malformed colors remain unresolved rather than being +guessed. Theme and indexed colors apply OOXML tint in HSL luminance space. +Spreadsheet font theme indexes use `0=lt1`, `1=dk1`, `2=lt2`, `3=dk2`, then +the accent slots; this intentionally differs from DrawingML `clrScheme` XML +order. `sysClr.lastClr` is only a fallback, and an xf with `applyFont=false` +does not contribute its `fontId`. + +Default Markdown and HTML projections add a `Text formatting` section whenever +a returned cell or rich-text run has strike-through or an explicit font color. +Conditional-format rendering is not evaluated, so it cannot supply a final +display color. + The corpus benchmark never writes the source workbooks and publishes no cell content. Human-edit simulations operate only on temporary sanitized fixtures or temporary workbook copies. diff --git a/crates/opsail-read/src/lib.rs b/crates/opsail-read/src/lib.rs index 665225b..37db3c3 100644 --- a/crates/opsail-read/src/lib.rs +++ b/crates/opsail-read/src/lib.rs @@ -25,11 +25,14 @@ pub use model::{ pub use opsail_chrome::{CdpSource, CdpWaitUntil, ChromeError, ChromeSource}; pub use xlsx::{ ArtifactKind, CellValueType, DateSystem, DefinedName, FormulaKind, SelectionBounds, SheetState, - WorkbookCell, WorkbookExtractionInfo, WorkbookExtractionMethod, WorkbookFeatureInventory, - WorkbookHyperlink, WorkbookInfo, WorkbookMetadata, WorkbookPartRevision, WorkbookReadResult, - WorkbookRevision, WorkbookRevisionDiff, WorkbookSelection, WorkbookSession, - WorkbookSessionMetrics, WorkbookSessionRefresh, WorkbookSheet, WorkbookStatistics, - WorksheetFeatureInventory, inspect_workbook_revision, merge_markdown_mirror, + WorkbookCell, WorkbookColumnOverflow, WorkbookExtractionInfo, WorkbookExtractionMethod, + WorkbookFeatureInventory, WorkbookFontColor, WorkbookHyperlink, WorkbookInfo, + WorkbookMergeMembership, WorkbookMergeRole, WorkbookMetadata, WorkbookPartRevision, + WorkbookPicture, WorkbookReadResult, WorkbookRevision, WorkbookRevisionDiff, + WorkbookRichTextRun, WorkbookRowOverflow, WorkbookSelection, WorkbookSelectionOverflow, + WorkbookSession, WorkbookSessionMetrics, WorkbookSessionRefresh, WorkbookSheet, + WorkbookStatistics, WorksheetFeatureInventory, WorksheetPageSetup, WorksheetPrintEvidence, + WorksheetPrintOptions, inspect_workbook_revision, merge_markdown_mirror, }; /// Auto-detect and read a supported document artifact. diff --git a/crates/opsail-read/src/model.rs b/crates/opsail-read/src/model.rs index 309cb77..88dddd6 100644 --- a/crates/opsail-read/src/model.rs +++ b/crates/opsail-read/src/model.rs @@ -115,7 +115,8 @@ pub struct SpreadsheetReadOptions { /// Repeated `Sheet!A1:D20` selectors. With no selectors, visible sheets get /// a bounded preview instead of an unbounded workbook dump. pub ranges: Vec, - /// Maximum number of non-empty cell records published across selections. + /// Maximum number of cell records published across selections, including + /// blank requested cells that carry merge-covered membership evidence. pub max_cells: usize, /// Maximum cumulative uncompressed OOXML bytes read from the ZIP package. pub max_expanded_bytes: usize, diff --git a/crates/opsail-read/src/xlsx.rs b/crates/opsail-read/src/xlsx.rs index a1205f5..18a9b55 100644 --- a/crates/opsail-read/src/xlsx.rs +++ b/crates/opsail-read/src/xlsx.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs::File; use std::io::{BufRead, BufReader, Read, Seek}; use std::path::{Path, PathBuf}; @@ -7,6 +7,7 @@ use std::time::SystemTime; use quick_xml::Reader; use quick_xml::events::{BytesRef, BytesStart, Event}; +use ring::digest::{SHA256, digest}; use serde::Serialize; use serde_json::{Value, json}; use url::Url; @@ -20,6 +21,12 @@ const XLSX_CONTENT_TYPE: &str = "application/vnd.openxmlformats-officedocument.s const PREVIEW_GRID_MAX_AREA: u64 = 2_000; const MAX_CELL_TEXT_BYTES: usize = 32_767 * 4; const MAX_FEATURE_REFERENCES: usize = 10_000; +const MAX_IMAGE_REFERENCES: usize = 256; +/// Raw image bytes are capped below the 16 MiB Host response ceiling. Base64 +/// expands the total to at most about 5.4 MiB, leaving room for cells and the +/// duplicated Markdown/HTML evidence. +const MAX_IMAGE_PAYLOAD_BYTES: usize = 2 * 1024 * 1024; +const MAX_TOTAL_IMAGE_PAYLOAD_BYTES: usize = 4 * 1024 * 1024; #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] @@ -209,6 +216,7 @@ pub struct WorkbookSession { struct WorkbookSessionContext { shared_strings: Vec, styles: Styles, + content_types: PackageContentTypes, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -261,6 +269,8 @@ impl WorkbookRevision { | "xl/sharedStrings.xml" | "xl/styles.xml" ) || name.starts_with("xl/theme/") + || name.starts_with("xl/drawings/") + || name.starts_with("xl/media/") }); WorkbookRevisionDiff { unchanged: changed_parts.is_empty(), @@ -457,8 +467,12 @@ pub struct WorkbookSheet { pub semantic_bounds_complete: bool, pub selected: bool, pub merged_ranges: Vec, + /// Worksheet pictures. Inventory entries never carry `dataUri`; pixel + /// payloads are published only on intersecting selections. + pub pictures: Vec, pub hidden_rows: usize, pub hidden_columns: usize, + pub print: WorksheetPrintEvidence, pub features: WorksheetFeatureInventory, } @@ -466,6 +480,11 @@ pub struct WorkbookSheet { #[serde(rename_all = "camelCase")] pub struct WorksheetFeatureInventory { pub scanned: bool, + /// True only when every cell element in `sheetData` was inspected. + pub cell_data_complete: bool, + /// True when parsing reached the end of the worksheet, including features + /// serialized after `sheetData`. + pub tail_features_complete: bool, pub complete: bool, pub feature_references_truncated: bool, pub formula_cells: usize, @@ -489,6 +508,48 @@ pub struct WorksheetFeatureInventory { pub controls: usize, } +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorksheetPrintEvidence { + #[serde(skip_serializing_if = "Option::is_none")] + pub print_area: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub print_titles: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub page_setup: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub print_options: Option, + pub row_breaks: Vec, + pub column_breaks: Vec, + pub header_footer: bool, +} + +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorksheetPageSetup { + #[serde(skip_serializing_if = "Option::is_none")] + pub orientation: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub paper_size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub fit_to_page: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub fit_to_width: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub fit_to_height: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub scale: Option, +} + +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorksheetPrintOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub grid_lines: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub headings: Option, +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] pub struct WorkbookHyperlink { @@ -527,10 +588,78 @@ pub struct WorkbookSelection { pub sheet: String, pub range: String, pub bounds: SelectionBounds, + #[serde(skip_serializing_if = "Option::is_none")] + pub used_bounds: Option, + pub merged_ranges: Vec, + /// Pictures whose anchors intersect this selection. Payloads are bounded + /// independently from the cell cap. + pub images: Vec, + pub images_truncated: bool, + #[serde(skip_serializing_if = "WorkbookSelectionOverflow::is_empty")] + pub overflow: WorkbookSelectionOverflow, pub cells: Vec, pub truncated: bool, } +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookPicture { + pub sheet: String, + pub from_cell: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub to_cell: Option, + /// Zero-based OOXML drawing marker row index. + pub from_row_index: u32, + /// Zero-based OOXML drawing marker column index. + pub from_column_index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub to_row_index: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub to_column_index: Option, + pub media_part: String, + pub content_type: String, + pub byte_size: usize, + pub sha256: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub data_uri: Option, + pub payload_truncated: bool, +} + +#[derive(Debug, Clone, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookSelectionOverflow { + #[serde(skip_serializing_if = "Option::is_none")] + pub left: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub right: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub above: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub below: Option, +} + +impl WorkbookSelectionOverflow { + fn is_empty(&self) -> bool { + self.left.is_none() && self.right.is_none() && self.above.is_none() && self.below.is_none() + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookColumnOverflow { + pub min_column: u16, + pub max_column: u16, + pub cell_count: usize, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookRowOverflow { + pub min_row: u32, + pub max_row: u32, + pub cell_count: usize, +} + #[derive(Debug, Clone, Copy, Serialize)] #[serde(rename_all = "camelCase")] pub struct SelectionBounds { @@ -559,11 +688,61 @@ pub struct WorkbookCell { pub shared_formula_index: Option, pub rich_text: bool, #[serde(skip_serializing_if = "Option::is_none")] + pub font_strike: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub font_color: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub rich_text_runs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub merge: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub style_index: Option, #[serde(skip_serializing_if = "Option::is_none")] pub number_format: Option, } +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookMergeMembership { + pub range: String, + pub anchor: String, + pub role: WorkbookMergeRole, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum WorkbookMergeRole { + Anchor, + Covered, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookFontColor { + #[serde(skip_serializing_if = "Option::is_none")] + pub rgb: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub theme: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub indexed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub auto: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tint: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub resolved_rgb: Option, +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WorkbookRichTextRun { + pub text: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub strike: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub font_color: Option, +} + #[derive(Debug, Clone, Copy, Serialize)] #[serde(rename_all = "kebab-case")] pub enum CellValueType { @@ -622,6 +801,9 @@ struct SelectionPlan { struct Styles { custom_formats: HashMap, cell_formats: Vec, + fonts: Vec, + cell_font_indexes: Vec>, + theme: Vec>, } impl Styles { @@ -632,12 +814,24 @@ impl Styles { .cloned() .or_else(|| builtin_number_format(id).map(str::to_owned)) } + + fn font(&self, style_index: Option) -> Option<&FontFormat> { + let font_index = (*self.cell_font_indexes.get(style_index?)?)?; + self.fonts.get(font_index) + } +} + +#[derive(Debug, Clone, Default)] +struct FontFormat { + strike: Option, + color: Option, } #[derive(Debug, Clone)] struct SharedString { text: String, rich: bool, + runs: Vec, } #[derive(Debug, Clone)] @@ -647,6 +841,52 @@ struct PartRelationship { external: bool, } +#[derive(Debug, Clone, Default)] +struct PackageContentTypes { + defaults: HashMap, + overrides: HashMap, +} + +impl PackageContentTypes { + fn for_part(&self, part: &str) -> Option { + let part = part.trim_start_matches('/'); + self.overrides.get(part).cloned().or_else(|| { + let extension = part.rsplit_once('.')?.1.to_ascii_lowercase(); + self.defaults.get(&extension).cloned() + }) + } +} + +#[derive(Debug, Clone, Copy, Default)] +struct DrawingMarker { + row: Option, + column: Option, +} + +#[derive(Debug, Default)] +struct DrawingAnchorBuilder { + from: DrawingMarker, + to: DrawingMarker, + two_cell: bool, + has_picture: bool, + inside_picture: bool, + image_relationship_id: Option, +} + +#[derive(Debug)] +struct DrawingPicture { + from: DrawingMarker, + to: Option, + media_part: String, +} + +#[derive(Debug, Clone)] +struct MediaAsset { + content_type: String, + bytes: Vec, + sha256: String, +} + #[derive(Debug, Default)] struct CellBuilder { reference: String, @@ -659,6 +899,9 @@ struct CellBuilder { formula_reference: Option, shared_formula_index: Option, rich_text: bool, + rich_text_runs: Vec, + current_run: Option, + inside_run_properties: bool, has_value: bool, has_inline: bool, has_formula: bool, @@ -672,8 +915,11 @@ struct SheetScan { dimension: Option, semantic: Option, merged_ranges: Vec, + drawing_relationship_ids: Vec, + pictures: Vec, hidden_rows: usize, hidden_columns: usize, + print: WorksheetPrintEvidence, cell_elements: usize, non_empty_cells: usize, style_only_cells: usize, @@ -783,6 +1029,15 @@ impl ArchiveReader { } fn read_xml(&mut self, name: &str) -> Result, ReadError> { + let Some(bytes) = self.read_bytes(name)? else { + return Ok(None); + }; + String::from_utf8(bytes) + .map(Some) + .map_err(|_| ReadError::InvalidXlsx(format!("OOXML part `{name}` is not UTF-8 XML"))) + } + + fn read_bytes(&mut self, name: &str) -> Result>, ReadError> { let mut entry = match self.archive.by_name(name) { Ok(entry) => entry, Err(ZipError::FileNotFound) => return Ok(None), @@ -825,9 +1080,7 @@ impl ArchiveReader { }); } self.expanded_bytes += bytes.len(); - String::from_utf8(bytes) - .map(Some) - .map_err(|_| ReadError::InvalidXlsx(format!("OOXML part `{name}` is not UTF-8 XML"))) + Ok(Some(bytes)) } #[allow(clippy::too_many_arguments)] @@ -958,7 +1211,7 @@ fn read_xlsx_incremental( ..WorkbookStatistics::default() }; let mut warnings = result.warnings.clone(); - let partial_warning = "incremental worksheet refresh is bounded to the cached selections; semantic bounds and worksheet feature inventory are partial"; + let partial_warning = "incremental worksheet refresh is bounded to the cached selections; later cell bodies and semantic bounds may be partial while worksheet tail features are still scanned"; if !warnings.iter().any(|warning| warning == partial_warning) { warnings.push(partial_warning.to_owned()); } @@ -986,6 +1239,18 @@ fn read_xlsx_incremental( .map(|selection| selection.cells.len()) .sum::(); let mut remaining_cells = options.spreadsheet.max_cells.saturating_sub(retained_cells); + let retained_image_payload_bytes = result + .workbook + .selections + .iter() + .filter(|selection| !changed_sheet_names.contains(&selection.sheet)) + .flat_map(|selection| &selection.images) + .filter(|picture| picture.data_uri.is_some()) + .map(|picture| picture.byte_size) + .sum::(); + let mut remaining_image_payload_bytes = + MAX_TOTAL_IMAGE_PAYLOAD_BYTES.saturating_sub(retained_image_payload_bytes); + let mut media_cache = HashMap::new(); let mut patches = Vec::new(); for part in changed_worksheet_parts { let sheet_index = result @@ -1008,6 +1273,11 @@ fn read_xlsx_incremental( .map(|(global_index, selection)| { let mut selection = selection.clone(); selection.cells.clear(); + selection.used_bounds = None; + selection.merged_ranges.clear(); + selection.images.clear(); + selection.images_truncated = false; + selection.overflow = WorkbookSelectionOverflow::default(); selection.truncated = false; (global_index, selection) }) @@ -1033,7 +1303,7 @@ fn read_xlsx_incremental( .map(|xml| parse_part_relationships(&xml)) .transpose()? .unwrap_or_default(); - let scan = package.scan_worksheet( + let mut scan = package.scan_worksheet( part, &plans, &mut local_selections, @@ -1045,6 +1315,20 @@ fn read_xlsx_incremental( &mut remaining_cells, &mut warnings, )?; + let (pictures, picture_references_truncated) = read_sheet_pictures( + &mut package, + &sheet_name, + part, + &scan.drawing_relationship_ids, + &relationships, + &context.content_types, + &mut local_selections, + &mut media_cache, + &mut remaining_image_payload_bytes, + &mut warnings, + )?; + scan.pictures = pictures; + scan.features.feature_references_truncated |= picture_references_truncated; statistics.scanned_sheets += 1; statistics.cell_elements += scan.cell_elements; statistics.non_empty_cells += scan.non_empty_cells; @@ -1078,10 +1362,12 @@ fn read_xlsx_incremental( let sheet = &mut result.workbook.sheets[patch.sheet_index]; sheet.declared_dimension = patch.scan.dimension; sheet.semantic_bounds = patch.scan.semantic.map(format_bounds); - sheet.semantic_bounds_complete = patch.scan.features.complete; + sheet.semantic_bounds_complete = patch.scan.features.cell_data_complete; sheet.merged_ranges = patch.scan.merged_ranges; + sheet.pictures = patch.scan.pictures; sheet.hidden_rows = patch.scan.hidden_rows; sheet.hidden_columns = patch.scan.hidden_columns; + apply_worksheet_print_scan(&mut sheet.print, patch.scan.print); sheet.features = patch.scan.features; } statistics.returned_cells = result @@ -1197,12 +1483,13 @@ fn read_xlsx_with_context( .iter() .map(|part| (part.name.as_str(), part)) .collect(); - let content_types = package.read_required_xml("[Content_Types].xml")?; - if !content_types.contains(XLSX_CONTENT_TYPE) { + let content_types_xml = package.read_required_xml("[Content_Types].xml")?; + if !content_types_xml.contains(XLSX_CONTENT_TYPE) { return Err(ReadError::InvalidXlsx( "package does not declare an XLSX workbook content type".to_owned(), )); } + let content_types = parse_content_types(&content_types_xml)?; let workbook_xml = package.read_required_xml("xl/workbook.xml")?; let relationships_xml = package.read_required_xml("xl/_rels/workbook.xml.rels")?; let relationships = parse_relationships(&relationships_xml)?; @@ -1215,16 +1502,22 @@ fn read_xlsx_with_context( let (shared_strings, styles) = if options.spreadsheet.revision_only { (Vec::new(), Styles::default()) } else { - let shared_strings = package + let mut shared_strings = package .read_xml("xl/sharedStrings.xml")? .map(|xml| parse_shared_strings(&xml)) .transpose()? .unwrap_or_default(); - let styles = package + let mut styles = package .read_xml("xl/styles.xml")? .map(|xml| parse_styles(&xml)) .transpose()? .unwrap_or_default(); + let theme = package + .read_xml("xl/theme/theme1.xml")? + .map(|xml| parse_theme_colors(&xml)) + .transpose()? + .unwrap_or_default(); + resolve_published_colors(&mut styles, &mut shared_strings, &theme); (shared_strings, styles) }; features.rich_string_items = shared_strings.iter().filter(|item| item.rich).count(); @@ -1264,8 +1557,18 @@ fn read_xlsx_with_context( semantic_bounds_complete: false, selected: plans.iter().any(|plan| plan.sheet_index == index), merged_ranges: Vec::new(), + pictures: Vec::new(), hidden_rows: 0, hidden_columns: 0, + print: WorksheetPrintEvidence { + print_area: print_defined_name(&workbook.defined_names, index, "_xlnm.Print_Area"), + print_titles: print_defined_name( + &workbook.defined_names, + index, + "_xlnm.Print_Titles", + ), + ..WorksheetPrintEvidence::default() + }, features: WorksheetFeatureInventory::default(), }) .collect(); @@ -1275,6 +1578,8 @@ fn read_xlsx_with_context( }; let mut warnings = Vec::new(); let mut remaining_cells = options.spreadsheet.max_cells; + let mut remaining_image_payload_bytes = MAX_TOTAL_IMAGE_PAYLOAD_BYTES; + let mut media_cache = HashMap::new(); let mut plans_by_sheet: BTreeMap> = BTreeMap::new(); for plan in plans { @@ -1299,7 +1604,7 @@ fn read_xlsx_with_context( .map(|xml| parse_part_relationships(&xml)) .transpose()? .unwrap_or_default(); - let scan = package.scan_worksheet( + let mut scan = package.scan_worksheet( &descriptor.path, &sheet_plans, &mut selections, @@ -1311,13 +1616,29 @@ fn read_xlsx_with_context( &mut remaining_cells, &mut warnings, )?; + let (pictures, picture_references_truncated) = read_sheet_pictures( + &mut package, + &descriptor.name, + &descriptor.path, + &scan.drawing_relationship_ids, + &sheet_relationships, + &content_types, + &mut selections, + &mut media_cache, + &mut remaining_image_payload_bytes, + &mut warnings, + )?; + scan.pictures = pictures; + scan.features.feature_references_truncated |= picture_references_truncated; let sheet = &mut sheets[sheet_index]; sheet.declared_dimension = scan.dimension; sheet.semantic_bounds = scan.semantic.map(format_bounds); - sheet.semantic_bounds_complete = scan.features.complete; + sheet.semantic_bounds_complete = scan.features.cell_data_complete; sheet.merged_ranges = scan.merged_ranges; + sheet.pictures = scan.pictures; sheet.hidden_rows = scan.hidden_rows; sheet.hidden_columns = scan.hidden_columns; + apply_worksheet_print_scan(&mut sheet.print, scan.print); sheet.features = scan.features; statistics.scanned_sheets += 1; statistics.cell_elements += scan.cell_elements; @@ -1337,12 +1658,13 @@ fn read_xlsx_with_context( options.spreadsheet.max_cells )); } - if sheets - .iter() - .any(|sheet| sheet.features.scanned && !sheet.features.complete) - { + if sheets.iter().any(|sheet| { + sheet.features.scanned + && !sheet.features.cell_data_complete + && sheet.features.tail_features_complete + }) { warnings.push( - "targeted worksheet scan stopped after the requested rows; semantic bounds and worksheet feature inventory are partial" + "targeted worksheet scan skipped cell bodies after the requested rows; semantic bounds and cell statistics are partial, but worksheet tail features were scanned" .to_owned(), ); } @@ -1403,6 +1725,7 @@ fn read_xlsx_with_context( WorkbookSessionContext { shared_strings, styles, + content_types, }, )) } @@ -1467,6 +1790,426 @@ fn parse_part_relationships(xml: &str) -> Result Result { + let mut reader = xml_reader(xml); + let mut content_types = PackageContentTypes::default(); + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"Default" => + { + let extension = attribute(&element, b"Extension")? + .ok_or_else(|| { + ReadError::InvalidXlsx("content-type default has no Extension".to_owned()) + })? + .to_ascii_lowercase(); + let content_type = attribute(&element, b"ContentType")?.ok_or_else(|| { + ReadError::InvalidXlsx("content-type default has no ContentType".to_owned()) + })?; + content_types.defaults.insert(extension, content_type); + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"Override" => + { + let part = attribute(&element, b"PartName")? + .ok_or_else(|| { + ReadError::InvalidXlsx("content-type override has no PartName".to_owned()) + })? + .trim_start_matches('/') + .to_owned(); + let content_type = attribute(&element, b"ContentType")?.ok_or_else(|| { + ReadError::InvalidXlsx("content-type override has no ContentType".to_owned()) + })?; + content_types.overrides.insert(part, content_type); + } + Event::Eof => break, + _ => {} + } + } + Ok(content_types) +} + +#[derive(Debug, Clone, Copy)] +enum DrawingMarkerKind { + From, + To, +} + +#[derive(Debug, Clone, Copy)] +enum DrawingMarkerField { + Row, + Column, +} + +fn parse_drawing_pictures( + xml: &str, + drawing_part: &str, + relationships: &HashMap, +) -> Result, ReadError> { + let mut reader = xml_reader(xml); + let mut pictures = Vec::new(); + let mut anchor: Option = None; + let mut marker_kind: Option = None; + let mut marker_field: Option = None; + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) + if matches!( + local_name(element.name().as_ref()), + b"twoCellAnchor" | b"oneCellAnchor" + ) => + { + anchor = Some(DrawingAnchorBuilder { + two_cell: local_name(element.name().as_ref()) == b"twoCellAnchor", + ..DrawingAnchorBuilder::default() + }); + } + Event::Start(element) + if anchor.is_some() && local_name(element.name().as_ref()) == b"from" => + { + marker_kind = Some(DrawingMarkerKind::From); + } + Event::Start(element) + if anchor.is_some() && local_name(element.name().as_ref()) == b"to" => + { + marker_kind = Some(DrawingMarkerKind::To); + } + Event::Start(element) + if marker_kind.is_some() && local_name(element.name().as_ref()) == b"row" => + { + marker_field = Some(DrawingMarkerField::Row); + } + Event::Start(element) + if marker_kind.is_some() && local_name(element.name().as_ref()) == b"col" => + { + marker_field = Some(DrawingMarkerField::Column); + } + Event::Text(text) + if anchor.is_some() && marker_kind.is_some() && marker_field.is_some() => + { + let value = text.decode().map_err(encoding_error)?; + if let Ok(value) = value.trim().parse::() { + let marker = match marker_kind.expect("marker kind is checked") { + DrawingMarkerKind::From => { + &mut anchor.as_mut().expect("anchor is checked").from + } + DrawingMarkerKind::To => { + &mut anchor.as_mut().expect("anchor is checked").to + } + }; + match marker_field.expect("marker field is checked") { + DrawingMarkerField::Row => marker.row = Some(value), + DrawingMarkerField::Column => marker.column = Some(value), + } + } + } + Event::Start(element) + if anchor.is_some() && local_name(element.name().as_ref()) == b"pic" => + { + let anchor = anchor.as_mut().expect("anchor is checked"); + anchor.has_picture = true; + anchor.inside_picture = true; + } + Event::Empty(element) + if anchor.is_some() && local_name(element.name().as_ref()) == b"pic" => + { + anchor.as_mut().expect("anchor is checked").has_picture = true; + } + Event::Start(element) | Event::Empty(element) + if anchor.as_ref().is_some_and(|anchor| anchor.inside_picture) + && local_name(element.name().as_ref()) == b"blip" => + { + if let Some(relationship_id) = attribute_exact(&element, b"r:embed")? { + anchor + .as_mut() + .expect("anchor is checked") + .image_relationship_id = Some(relationship_id); + } + } + Event::End(element) + if matches!(local_name(element.name().as_ref()), b"row" | b"col") => + { + marker_field = None; + } + Event::End(element) + if matches!(local_name(element.name().as_ref()), b"from" | b"to") => + { + marker_kind = None; + } + Event::End(element) if local_name(element.name().as_ref()) == b"pic" => { + if let Some(anchor) = anchor.as_mut() { + anchor.inside_picture = false; + } + } + Event::End(element) + if matches!( + local_name(element.name().as_ref()), + b"twoCellAnchor" | b"oneCellAnchor" + ) => + { + if let Some(anchor) = anchor.take() + && anchor.has_picture + && anchor.from.row.is_some() + && anchor.from.column.is_some() + && (!anchor.two_cell || (anchor.to.row.is_some() && anchor.to.column.is_some())) + && let Some(relationship_id) = anchor.image_relationship_id + && let Some(relationship) = relationships.get(&relationship_id) + && relationship.relationship_type.ends_with("/image") + && !relationship.external + { + pictures.push(DrawingPicture { + from: anchor.from, + to: anchor.two_cell.then_some(anchor.to), + media_part: resolve_part_target(drawing_part, &relationship.target)?, + }); + } + marker_kind = None; + marker_field = None; + } + Event::Eof => break, + _ => {} + } + } + Ok(pictures) +} + +#[allow(clippy::too_many_arguments)] +fn read_sheet_pictures( + package: &mut ArchiveReader, + sheet_name: &str, + sheet_part: &str, + drawing_relationship_ids: &[String], + sheet_relationships: &HashMap, + content_types: &PackageContentTypes, + selections: &mut [WorkbookSelection], + media_cache: &mut HashMap, + remaining_payload_bytes: &mut usize, + warnings: &mut Vec, +) -> Result<(Vec, bool), ReadError> { + let mut inventory = Vec::new(); + let mut inventory_truncated = false; + for relationship_id in drawing_relationship_ids { + let Some(relationship) = sheet_relationships + .get(relationship_id) + .filter(|relationship| { + relationship.relationship_type.ends_with("/drawing") && !relationship.external + }) + else { + continue; + }; + let drawing_part = resolve_part_target(sheet_part, &relationship.target)?; + let drawing_xml = package.read_required_xml(&drawing_part)?; + let drawing_relationships = package + .read_xml(&part_relationship_path(&drawing_part)?)? + .map(|xml| parse_part_relationships(&xml)) + .transpose()? + .unwrap_or_default(); + let drawing_pictures = + parse_drawing_pictures(&drawing_xml, &drawing_part, &drawing_relationships)?; + for drawing_picture in drawing_pictures { + let Some(from_cell) = drawing_marker_cell(drawing_picture.from) else { + push_unique_warning( + warnings, + format!("worksheet `{sheet_name}` has a picture with an invalid from marker"), + ); + continue; + }; + let to_cell = drawing_picture.to.and_then(drawing_marker_cell); + let inventory_slot = inventory.len() < MAX_IMAGE_REFERENCES; + if !inventory_slot { + inventory_truncated = true; + } + let mut selection_indexes = Vec::new(); + for (index, selection) in selections.iter_mut().enumerate() { + if selection.sheet != sheet_name + || !drawing_picture_intersects_selection(&drawing_picture, selection.bounds) + { + continue; + } + if selection.images.len() >= MAX_IMAGE_REFERENCES { + selection.images_truncated = true; + push_unique_warning( + warnings, + format!( + "worksheet image inventory was truncated at {MAX_IMAGE_REFERENCES} pictures per selection" + ), + ); + } else { + selection_indexes.push(index); + } + } + if !inventory_slot && selection_indexes.is_empty() { + continue; + } + if !media_cache.contains_key(&drawing_picture.media_part) { + let bytes = package + .read_bytes(&drawing_picture.media_part)? + .ok_or_else(|| { + ReadError::InvalidXlsx(format!( + "drawing image part `{}` is missing", + drawing_picture.media_part + )) + })?; + let content_type = content_types + .for_part(&drawing_picture.media_part) + .unwrap_or_else(|| "application/octet-stream".to_owned()); + let sha256 = hex_lower(digest(&SHA256, &bytes).as_ref()); + media_cache.insert( + drawing_picture.media_part.clone(), + MediaAsset { + content_type, + bytes, + sha256, + }, + ); + } + let asset = media_cache + .get(&drawing_picture.media_part) + .expect("media asset was inserted"); + let picture = WorkbookPicture { + sheet: sheet_name.to_owned(), + from_cell, + to_cell, + from_row_index: drawing_picture.from.row.unwrap_or_default(), + from_column_index: drawing_picture.from.column.unwrap_or_default(), + to_row_index: drawing_picture.to.and_then(|marker| marker.row), + to_column_index: drawing_picture.to.and_then(|marker| marker.column), + media_part: drawing_picture.media_part, + content_type: asset.content_type.clone(), + byte_size: asset.bytes.len(), + sha256: asset.sha256.clone(), + data_uri: None, + payload_truncated: false, + }; + if inventory_slot { + inventory.push(picture.clone()); + } + for index in selection_indexes { + let selection = &mut selections[index]; + let mut selected_picture = picture.clone(); + if asset.bytes.len() <= MAX_IMAGE_PAYLOAD_BYTES + && asset.bytes.len() <= *remaining_payload_bytes + { + selected_picture.data_uri = Some(format!( + "data:{};base64,{}", + asset.content_type, + encode_base64(&asset.bytes) + )); + *remaining_payload_bytes -= asset.bytes.len(); + } else { + selected_picture.payload_truncated = true; + selection.images_truncated = true; + push_unique_warning( + warnings, + format!( + "worksheet image payloads are limited to {MAX_IMAGE_PAYLOAD_BYTES} bytes per image and {MAX_TOTAL_IMAGE_PAYLOAD_BYTES} bytes total; omitted payloads remain available as metadata" + ), + ); + } + selection.images.push(selected_picture); + } + } + } + if inventory_truncated { + push_unique_warning( + warnings, + format!( + "worksheet picture inventory was truncated at {MAX_IMAGE_REFERENCES} pictures per scanned sheet" + ), + ); + } + Ok((inventory, inventory_truncated)) +} + +fn drawing_marker_cell(marker: DrawingMarker) -> Option { + let row = marker.row?.checked_add(1)?; + let column = marker.column?.checked_add(1)?; + if row > 1_048_576 || column > 16_384 { + return None; + } + Some(format!( + "{}{}", + column_name(u16::try_from(column).ok()?), + row + )) +} + +fn drawing_picture_intersects_selection(picture: &DrawingPicture, bounds: SelectionBounds) -> bool { + drawing_anchor_intersects_selection(picture.from, picture.to, bounds) +} + +fn drawing_anchor_intersects_selection( + from: DrawingMarker, + to: Option, + bounds: SelectionBounds, +) -> bool { + let Some(from_row) = from.row.map(|value| value.saturating_add(1)) else { + return false; + }; + let Some(from_column) = from.column.map(|value| value.saturating_add(1)) else { + return false; + }; + let to_row = to + .and_then(|marker| marker.row) + .unwrap_or(from_row - 1) + .saturating_add(1); + let to_column = to + .and_then(|marker| marker.column) + .unwrap_or(from_column - 1) + .saturating_add(1); + let start_row = from_row.min(to_row); + let end_row = from_row.max(to_row); + let start_column = from_column.min(to_column); + let end_column = from_column.max(to_column); + start_row <= bounds.end_row + && end_row >= bounds.start_row + && start_column <= u32::from(bounds.end_column) + && end_column >= u32::from(bounds.start_column) +} + +fn encode_base64(bytes: &[u8]) -> String { + const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut output = String::with_capacity(bytes.len().div_ceil(3) * 4); + for chunk in bytes.chunks(3) { + let first = chunk[0]; + let second = chunk.get(1).copied().unwrap_or(0); + let third = chunk.get(2).copied().unwrap_or(0); + output.push(char::from(TABLE[usize::from(first >> 2)])); + output.push(char::from( + TABLE[usize::from(((first & 0x03) << 4) | (second >> 4))], + )); + if chunk.len() > 1 { + output.push(char::from( + TABLE[usize::from(((second & 0x0f) << 2) | (third >> 6))], + )); + } else { + output.push('='); + } + if chunk.len() > 2 { + output.push(char::from(TABLE[usize::from(third & 0x3f)])); + } else { + output.push('='); + } + } + output +} + +fn hex_lower(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for byte in bytes { + output.push(char::from(HEX[usize::from(byte >> 4)])); + output.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + output +} + +fn push_unique_warning(warnings: &mut Vec, warning: String) { + if !warnings.iter().any(|existing| existing == &warning) { + warnings.push(warning); + } +} + fn parse_workbook( xml: &str, relationships: &HashMap, @@ -1549,11 +2292,35 @@ fn parse_workbook( }) } +fn print_defined_name( + defined_names: &[DefinedName], + sheet_index: usize, + name: &str, +) -> Option { + defined_names + .iter() + .find(|defined_name| { + defined_name.name == name && defined_name.local_sheet_index == Some(sheet_index) + }) + .map(|defined_name| defined_name.reference.clone()) +} + +fn apply_worksheet_print_scan(target: &mut WorksheetPrintEvidence, scan: WorksheetPrintEvidence) { + target.page_setup = scan.page_setup; + target.print_options = scan.print_options; + target.row_breaks = scan.row_breaks; + target.column_breaks = scan.column_breaks; + target.header_footer = scan.header_footer; +} + fn parse_shared_strings(xml: &str) -> Result, ReadError> { let mut reader = xml_reader(xml); let mut strings = Vec::new(); let mut current: Option = None; let mut rich = false; + let mut runs = Vec::new(); + let mut run: Option = None; + let mut inside_run_properties = false; let mut inside_text = false; let mut phonetic_depth = 0usize; loop { @@ -1561,10 +2328,33 @@ fn parse_shared_strings(xml: &str) -> Result, ReadError> { Event::Start(element) if local_name(element.name().as_ref()) == b"si" => { current = Some(String::new()); rich = false; + runs.clear(); } Event::Start(element) if local_name(element.name().as_ref()) == b"r" => { if current.is_some() { rich = true; + run = Some(WorkbookRichTextRun::default()); + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"rPr" => { + inside_run_properties = true; + } + Event::Start(element) | Event::Empty(element) + if inside_run_properties && local_name(element.name().as_ref()) == b"strike" => + { + if let Some(run) = run.as_mut() { + run.strike = Some( + attribute(&element, b"val")? + .as_deref() + .is_none_or(xml_truthy), + ); + } + } + Event::Start(element) | Event::Empty(element) + if inside_run_properties && local_name(element.name().as_ref()) == b"color" => + { + if let Some(run) = run.as_mut() { + run.font_color = parse_font_color(&element)?; } } Event::Start(element) if local_name(element.name().as_ref()) == b"rPh" => { @@ -1577,12 +2367,27 @@ fn parse_shared_strings(xml: &str) -> Result, ReadError> { } Event::Text(text) if inside_text => { if let Some(value) = current.as_mut() { - push_bounded(value, &text.decode().map_err(encoding_error)?); + let decoded = text.decode().map_err(encoding_error)?; + push_bounded(value, &decoded); + if let Some(run) = run.as_mut() { + push_bounded(&mut run.text, &decoded); + } } } Event::GeneralRef(entity) if inside_text => { if let Some(value) = current.as_mut() { push_bounded(value, &decode_reference(&entity)?); + if let Some(run) = run.as_mut() { + push_bounded(&mut run.text, &decode_reference(&entity)?); + } + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"rPr" => { + inside_run_properties = false; + } + Event::End(element) if local_name(element.name().as_ref()) == b"r" => { + if let Some(run) = run.take() { + runs.push(run); } } Event::End(element) if local_name(element.name().as_ref()) == b"t" => { @@ -1595,6 +2400,7 @@ fn parse_shared_strings(xml: &str) -> Result, ReadError> { strings.push(SharedString { text: current.take().unwrap_or_default(), rich, + runs: std::mem::take(&mut runs), }); } Event::Eof => break, @@ -1608,6 +2414,8 @@ fn parse_styles(xml: &str) -> Result { let mut reader = xml_reader(xml); let mut styles = Styles::default(); let mut inside_cell_xfs = false; + let mut inside_fonts = false; + let mut font: Option = None; loop { match reader.read_event().map_err(xml_error)? { Event::Start(element) | Event::Empty(element) @@ -1623,6 +2431,41 @@ fn parse_styles(xml: &str) -> Result { Event::Start(element) if local_name(element.name().as_ref()) == b"cellXfs" => { inside_cell_xfs = true; } + Event::Start(element) if local_name(element.name().as_ref()) == b"fonts" => { + inside_fonts = true; + } + Event::End(element) if local_name(element.name().as_ref()) == b"fonts" => { + inside_fonts = false; + } + Event::Start(element) + if inside_fonts && local_name(element.name().as_ref()) == b"font" => + { + font = Some(FontFormat::default()); + } + Event::Empty(element) + if inside_fonts && local_name(element.name().as_ref()) == b"font" => + { + styles.fonts.push(FontFormat::default()); + } + Event::Start(element) | Event::Empty(element) + if font.is_some() && local_name(element.name().as_ref()) == b"strike" => + { + font.as_mut().unwrap().strike = Some( + attribute(&element, b"val")? + .as_deref() + .is_none_or(xml_truthy), + ); + } + Event::Start(element) | Event::Empty(element) + if font.is_some() && local_name(element.name().as_ref()) == b"color" => + { + font.as_mut().unwrap().color = parse_font_color(&element)?; + } + Event::End(element) if local_name(element.name().as_ref()) == b"font" => { + if let Some(font) = font.take() { + styles.fonts.push(font); + } + } Event::End(element) if local_name(element.name().as_ref()) == b"cellXfs" => { inside_cell_xfs = false; } @@ -1633,6 +2476,14 @@ fn parse_styles(xml: &str) -> Result { .and_then(|value| value.parse().ok()) .unwrap_or(0); styles.cell_formats.push(id); + let apply_font = attribute(&element, b"applyFont")? + .as_deref() + .map(xml_truthy); + let font_index = + attribute(&element, b"fontId")?.and_then(|value| value.parse().ok()); + styles + .cell_font_indexes + .push((apply_font != Some(false)).then_some(font_index).flatten()); } Event::Eof => break, _ => {} @@ -1641,6 +2492,193 @@ fn parse_styles(xml: &str) -> Result { Ok(styles) } +fn parse_font_color(element: &BytesStart<'_>) -> Result, ReadError> { + let color = WorkbookFontColor { + rgb: attribute(element, b"rgb")?.map(|value| value.to_ascii_uppercase()), + theme: attribute(element, b"theme")?.and_then(|value| value.parse().ok()), + indexed: attribute(element, b"indexed")?.and_then(|value| value.parse().ok()), + auto: attribute(element, b"auto")?.as_deref().map(xml_truthy), + tint: attribute(element, b"tint")?.and_then(|value| value.parse().ok()), + resolved_rgb: None, + }; + Ok((color.rgb.is_some() + || color.theme.is_some() + || color.indexed.is_some() + || color.auto.is_some() + || color.tint.is_some()) + .then_some(color)) +} + +fn parse_theme_colors(xml: &str) -> Result>, ReadError> { + let mut reader = xml_reader(xml); + let mut colors = vec![None; 12]; + let mut inside_scheme = false; + let mut slot = None; + loop { + match reader.read_event().map_err(xml_error)? { + Event::Start(element) if local_name(element.name().as_ref()) == b"clrScheme" => { + inside_scheme = true; + } + Event::End(element) if local_name(element.name().as_ref()) == b"clrScheme" => break, + Event::Start(element) if inside_scheme && slot.is_none() => { + slot = theme_color_index(local_name(element.name().as_ref())); + } + Event::Start(element) | Event::Empty(element) + if inside_scheme + && slot.is_some() + && matches!(local_name(element.name().as_ref()), b"srgbClr" | b"sysClr") => + { + let value = if local_name(element.name().as_ref()) == b"sysClr" { + attribute(&element, b"val")? + .filter(|value| is_rgb_hex(value)) + .or(attribute(&element, b"lastClr")?) + } else { + attribute(&element, b"val")? + }; + if let (Some(index), Some(value)) = (slot, value) { + colors[index] = Some(value.to_ascii_uppercase()); + } + } + Event::End(element) + if inside_scheme + && theme_color_index(local_name(element.name().as_ref())).is_some() => + { + slot = None; + } + Event::Eof => break, + _ => {} + } + } + Ok(colors) +} + +fn theme_color_index(name: &[u8]) -> Option { + match name { + // SpreadsheetML theme indexes deliberately swap the first two pairs + // from DrawingML clrScheme order: 0=lt1, 1=dk1, 2=lt2, 3=dk2. + b"lt1" => Some(0), + b"dk1" => Some(1), + b"lt2" => Some(2), + b"dk2" => Some(3), + b"accent1" => Some(4), + b"accent2" => Some(5), + b"accent3" => Some(6), + b"accent4" => Some(7), + b"accent5" => Some(8), + b"accent6" => Some(9), + b"hlink" => Some(10), + b"folHlink" => Some(11), + _ => None, + } +} + +fn is_rgb_hex(value: &str) -> bool { + value.len() == 6 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn resolve_published_colors( + styles: &mut Styles, + shared_strings: &mut [SharedString], + theme: &[Option], +) { + styles.theme = theme.to_vec(); + for font in &mut styles.fonts { + if let Some(color) = &mut font.color { + color.resolved_rgb = resolve_color(color, theme); + } + } + for item in shared_strings { + for run in &mut item.runs { + if let Some(color) = &mut run.font_color { + color.resolved_rgb = resolve_color(color, theme); + } + } + } +} + +fn resolve_color(color: &WorkbookFontColor, theme: &[Option]) -> Option { + if color.auto == Some(true) { + return None; + } + let base = if let Some(rgb) = &color.rgb { + (rgb.len() >= 6).then(|| rgb[rgb.len() - 6..].to_ascii_uppercase()) + } else if let Some(index) = color.theme { + theme.get(index as usize).cloned().flatten() + } else if let Some(index) = color.indexed { + indexed_color(index).map(str::to_owned) + } else { + None + }?; + apply_tint(&base, color.tint.unwrap_or(0.0)) +} + +fn indexed_color(index: u32) -> Option<&'static str> { + const COLORS: [&str; 64] = [ + "000000", "FFFFFF", "FF0000", "00FF00", "0000FF", "FFFF00", "FF00FF", "00FFFF", "000000", + "FFFFFF", "FF0000", "00FF00", "0000FF", "FFFF00", "FF00FF", "00FFFF", "800000", "008000", + "000080", "808000", "800080", "008080", "C0C0C0", "808080", "9999FF", "993366", "FFFFCC", + "CCFFFF", "660066", "FF8080", "0066CC", "CCCCFF", "000080", "FF00FF", "FFFF00", "00FFFF", + "800080", "800000", "008080", "0000FF", "00CCFF", "CCFFFF", "CCFFCC", "FFFF99", "99CCFF", + "FF99CC", "CC99FF", "FFCC99", "3366FF", "33CCCC", "99CC00", "FFCC00", "FF9900", "FF6600", + "666699", "969696", "003366", "339966", "003300", "333300", "993300", "993366", "333399", + "333333", + ]; + COLORS.get(index as usize).copied() +} + +fn apply_tint(rgb: &str, tint: f64) -> Option { + if rgb.len() != 6 || !(-1.0..=1.0).contains(&tint) { + return None; + } + let r = u8::from_str_radix(&rgb[0..2], 16).ok()? as f64 / 255.0; + let g = u8::from_str_radix(&rgb[2..4], 16).ok()? as f64 / 255.0; + let b = u8::from_str_radix(&rgb[4..6], 16).ok()? as f64 / 255.0; + let max = r.max(g).max(b); + let min = r.min(g).min(b); + let mut h = 0.0; + let l = (max + min) / 2.0; + let d = max - min; + let s = if d == 0.0 { + 0.0 + } else { + d / (1.0 - (2.0 * l - 1.0).abs()) + }; + if d != 0.0 { + h = if max == r { + ((g - b) / d) % 6.0 + } else if max == g { + (b - r) / d + 2.0 + } else { + (r - g) / d + 4.0 + } / 6.0; + if h < 0.0 { + h += 1.0; + } + } + let l = if tint < 0.0 { + l * (1.0 + tint) + } else { + l * (1.0 - tint) + tint + }; + let c = (1.0 - (2.0 * l - 1.0).abs()) * s; + let x = c * (1.0 - (((h * 6.0) % 2.0) - 1.0).abs()); + let m = l - c / 2.0; + let (r1, g1, b1) = match (h * 6.0).floor() as i32 { + 0 => (c, x, 0.0), + 1 => (x, c, 0.0), + 2 => (0.0, c, x), + 3 => (0.0, x, c), + 4 => (x, 0.0, c), + _ => (c, 0.0, x), + }; + Some(format!( + "{:02X}{:02X}{:02X}", + ((r1 + m) * 255.0).round() as u8, + ((g1 + m) * 255.0).round() as u8, + ((b1 + m) * 255.0).round() as u8 + )) +} + #[allow(clippy::too_many_arguments)] fn parse_worksheet( input: R, @@ -1659,12 +2697,14 @@ fn parse_worksheet( let mut buffer = Vec::new(); let mut scan = SheetScan::default(); scan.features.scanned = true; - scan.features.complete = true; + scan.features.cell_data_complete = true; let mut cell: Option = None; - let allow_early_stop = !options.ranges.is_empty(); + let allow_cell_skip = !options.ranges.is_empty(); let max_requested_row = plans.iter().map(|plan| plan.bounds.end_row).max(); - let mut declared_end_row = None; - let mut current_row: Option = None; + let mut selection_extents = vec![None; selections.len()]; + let mut inside_sheet_data = false; + let mut inside_row_breaks = false; + let mut inside_column_breaks = false; let mut phonetic_depth = 0usize; loop { match reader.read_event_into(&mut buffer).map_err(xml_error)? { @@ -1715,6 +2755,38 @@ fn parse_worksheet( && cell.cell_type.as_deref() == Some("inlineStr") { cell.rich_text = true; + cell.current_run = Some(WorkbookRichTextRun::default()); + } + } + Event::Start(element) if local_name(element.name().as_ref()) == b"rPr" => { + if let Some(cell) = cell.as_mut() + && cell.current_run.is_some() + { + cell.inside_run_properties = true; + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"strike" => + { + if let Some(cell) = cell.as_mut() + && cell.inside_run_properties + && let Some(run) = cell.current_run.as_mut() + { + run.strike = Some( + attribute(&element, b"val")? + .as_deref() + .is_none_or(xml_truthy), + ); + } + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"color" => + { + if let Some(cell) = cell.as_mut() + && cell.inside_run_properties + && let Some(run) = cell.current_run.as_mut() + { + run.font_color = parse_font_color(&element)?; } } Event::Start(element) if local_name(element.name().as_ref()) == b"rPh" => { @@ -1729,6 +2801,9 @@ fn parse_worksheet( push_bounded(&mut cell.formula, &text); } else if cell.capture_inline_text { push_bounded(&mut cell.inline, &text); + if let Some(run) = cell.current_run.as_mut() { + push_bounded(&mut run.text, &text); + } } } } @@ -1741,6 +2816,9 @@ fn parse_worksheet( push_bounded(&mut cell.formula, &text); } else if cell.capture_inline_text { push_bounded(&mut cell.inline, &text); + if let Some(run) = cell.current_run.as_mut() { + push_bounded(&mut run.text, &text); + } } } } @@ -1762,6 +2840,18 @@ fn parse_worksheet( Event::End(element) if local_name(element.name().as_ref()) == b"rPh" => { phonetic_depth = phonetic_depth.saturating_sub(1); } + Event::End(element) if local_name(element.name().as_ref()) == b"rPr" => { + if let Some(cell) = cell.as_mut() { + cell.inside_run_properties = false; + } + } + Event::End(element) if local_name(element.name().as_ref()) == b"r" => { + if let Some(cell) = cell.as_mut() + && let Some(run) = cell.current_run.take() + { + cell.rich_text_runs.push(run); + } + } Event::End(element) if local_name(element.name().as_ref()) == b"c" => { if let Some(builder) = cell.take() { if let Some(parsed) = finish_cell( @@ -1775,6 +2865,13 @@ fn parse_worksheet( scan.non_empty_cells += 1; update_semantic_bounds(&mut scan.semantic, parsed.row, parsed.column); for plan in plans { + record_selection_extent( + &mut selection_extents[plan.output_index], + &mut selections[plan.output_index].overflow, + plan.bounds, + parsed.row, + parsed.column, + ); if contains(plan.bounds, parsed.row, parsed.column) { let selection = &mut selections[plan.output_index]; if *remaining_cells > 0 { @@ -1794,12 +2891,6 @@ fn parse_worksheet( if local_name(element.name().as_ref()) == b"dimension" => { scan.dimension = attribute(&element, b"ref")?; - declared_end_row = scan.dimension.as_deref().and_then(|dimension| { - dimension.rsplit_once(':').map_or_else( - || parse_cell_reference(dimension).map(|(row, _)| row), - |(_, end)| parse_cell_reference(end).map(|(row, _)| row), - ) - }); } Event::Start(element) | Event::Empty(element) if local_name(element.name().as_ref()) == b"mergeCell" => @@ -1808,34 +2899,29 @@ fn parse_worksheet( scan.merged_ranges.push(reference); } } - Event::Start(element) | Event::Empty(element) - if local_name(element.name().as_ref()) == b"row" => - { - current_row = attribute(&element, b"r")?.and_then(|value| value.parse().ok()); - if allow_early_stop - && let (Some(row), Some(max_row), Some(sheet_end_row)) = - (current_row, max_requested_row, declared_end_row) + Event::Start(element) if local_name(element.name().as_ref()) == b"sheetData" => { + inside_sheet_data = true; + } + Event::End(element) if local_name(element.name().as_ref()) == b"sheetData" => { + inside_sheet_data = false; + } + Event::Start(element) if local_name(element.name().as_ref()) == b"row" => { + let row = record_row_metadata(&element, &mut scan)?; + if inside_sheet_data + && allow_cell_skip + && let (Some(row), Some(max_row)) = (row, max_requested_row) && row > max_row - && sheet_end_row > max_row - { - scan.features.complete = false; - break; - } - if attribute(&element, b"hidden")? - .as_deref() - .is_some_and(xml_truthy) { - scan.hidden_rows += 1; - } - let outline_level = attribute(&element, b"outlineLevel")? - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - if outline_level > 0 { - scan.features.outlined_rows += 1; - scan.features.max_row_outline_level = - scan.features.max_row_outline_level.max(outline_level); + scan.features.cell_data_complete = false; + let mut skipped = Vec::new(); + reader + .read_to_end_into(element.name(), &mut skipped) + .map_err(xml_error)?; } } + Event::Empty(element) if local_name(element.name().as_ref()) == b"row" => { + record_row_metadata(&element, &mut scan)?; + } Event::Start(element) | Event::Empty(element) if local_name(element.name().as_ref()) == b"col" => { @@ -1927,6 +3013,13 @@ fn parse_worksheet( if local_name(element.name().as_ref()) == b"drawing" => { scan.features.drawing_parts += 1; + if let Some(relationship_id) = attribute_exact(&element, b"r:id")? { + push_feature_reference( + &mut scan.drawing_relationship_ids, + relationship_id, + &mut scan.features.feature_references_truncated, + ); + } } Event::Start(element) | Event::Empty(element) if local_name(element.name().as_ref()) == b"legacyDrawing" => @@ -1934,10 +3027,45 @@ fn parse_worksheet( scan.features.comment_drawing_parts += 1; } Event::Start(element) | Event::Empty(element) - if matches!( - local_name(element.name().as_ref()), - b"pageSetup" | b"pageMargins" | b"printOptions" - ) => + if local_name(element.name().as_ref()) == b"pageSetUpPr" => + { + scan.features.page_setup = true; + let page_setup = scan.print.page_setup.get_or_insert_default(); + page_setup.fit_to_page = attribute(&element, b"fitToPage")? + .as_deref() + .map(xml_truthy); + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"pageSetup" => + { + scan.features.page_setup = true; + let page_setup = scan.print.page_setup.get_or_insert_default(); + page_setup.orientation = attribute(&element, b"orientation")?; + page_setup.paper_size = + attribute(&element, b"paperSize")?.and_then(|value| value.parse().ok()); + if let Some(value) = attribute(&element, b"fitToPage")? { + page_setup.fit_to_page = Some(xml_truthy(&value)); + } + page_setup.fit_to_width = + attribute(&element, b"fitToWidth")?.and_then(|value| value.parse().ok()); + page_setup.fit_to_height = + attribute(&element, b"fitToHeight")?.and_then(|value| value.parse().ok()); + page_setup.scale = + attribute(&element, b"scale")?.and_then(|value| value.parse().ok()); + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"printOptions" => + { + scan.features.page_setup = true; + scan.print.print_options = Some(WorksheetPrintOptions { + grid_lines: attribute(&element, b"gridLines")? + .as_deref() + .map(xml_truthy), + headings: attribute(&element, b"headings")?.as_deref().map(xml_truthy), + }); + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"pageMargins" => { scan.features.page_setup = true; } @@ -1945,6 +3073,40 @@ fn parse_worksheet( if local_name(element.name().as_ref()) == b"headerFooter" => { scan.features.header_footer = true; + scan.print.header_footer = true; + } + Event::Start(element) if local_name(element.name().as_ref()) == b"rowBreaks" => { + inside_row_breaks = true; + } + Event::End(element) if local_name(element.name().as_ref()) == b"rowBreaks" => { + inside_row_breaks = false; + } + Event::Start(element) if local_name(element.name().as_ref()) == b"colBreaks" => { + inside_column_breaks = true; + } + Event::End(element) if local_name(element.name().as_ref()) == b"colBreaks" => { + inside_column_breaks = false; + } + Event::Start(element) | Event::Empty(element) + if local_name(element.name().as_ref()) == b"brk" => + { + if let Some(reference) = + attribute(&element, b"id")?.and_then(|value| value.parse::().ok()) + { + if inside_row_breaks { + push_feature_reference( + &mut scan.print.row_breaks, + reference, + &mut scan.features.feature_references_truncated, + ); + } else if inside_column_breaks { + push_feature_reference( + &mut scan.print.column_breaks, + reference, + &mut scan.features.feature_references_truncated, + ); + } + } } Event::Start(element) | Event::Empty(element) if local_name(element.name().as_ref()) == b"sparkline" => @@ -1956,26 +3118,234 @@ fn parse_worksheet( { scan.features.controls += 1; } - Event::End(element) if local_name(element.name().as_ref()) == b"row" => { - if allow_early_stop - && let (Some(row), Some(max_row), Some(sheet_end_row)) = - (current_row, max_requested_row, declared_end_row) - && row >= max_row - && sheet_end_row > max_row - { - scan.features.complete = false; - break; - } - current_row = None; + Event::Eof => { + scan.features.tail_features_complete = true; + break; } - Event::Eof => break, _ => {} } buffer.clear(); } + scan.features.complete = + scan.features.cell_data_complete && scan.features.tail_features_complete; + for plan in plans { + selections[plan.output_index].used_bounds = + selection_extents[plan.output_index].map(format_bounds); + } + publish_selection_merges(plans, selections, &scan.merged_ranges, remaining_cells); Ok(scan) } +fn record_row_metadata( + element: &BytesStart<'_>, + scan: &mut SheetScan, +) -> Result, ReadError> { + let row = attribute(element, b"r")?.and_then(|value| value.parse().ok()); + if attribute(element, b"hidden")? + .as_deref() + .is_some_and(xml_truthy) + { + scan.hidden_rows += 1; + } + let outline_level = attribute(element, b"outlineLevel")? + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + if outline_level > 0 { + scan.features.outlined_rows += 1; + scan.features.max_row_outline_level = + scan.features.max_row_outline_level.max(outline_level); + } + Ok(row) +} + +fn record_selection_extent( + used_bounds: &mut Option, + overflow: &mut WorkbookSelectionOverflow, + requested: SelectionBounds, + row: u32, + column: u16, +) { + if (requested.start_row..=requested.end_row).contains(&row) { + update_semantic_bounds(used_bounds, row, column); + if column < requested.start_column { + update_column_overflow(&mut overflow.left, column); + } else if column > requested.end_column { + update_column_overflow(&mut overflow.right, column); + } + } else if (requested.start_column..=requested.end_column).contains(&column) { + if row < requested.start_row { + update_row_overflow(&mut overflow.above, row); + } else if row > requested.end_row { + update_row_overflow(&mut overflow.below, row); + } + } +} + +fn update_column_overflow(overflow: &mut Option, column: u16) { + if let Some(overflow) = overflow { + overflow.min_column = overflow.min_column.min(column); + overflow.max_column = overflow.max_column.max(column); + overflow.cell_count += 1; + } else { + *overflow = Some(WorkbookColumnOverflow { + min_column: column, + max_column: column, + cell_count: 1, + }); + } +} + +fn update_row_overflow(overflow: &mut Option, row: u32) { + if let Some(overflow) = overflow { + overflow.min_row = overflow.min_row.min(row); + overflow.max_row = overflow.max_row.max(row); + overflow.cell_count += 1; + } else { + *overflow = Some(WorkbookRowOverflow { + min_row: row, + max_row: row, + cell_count: 1, + }); + } +} + +fn publish_selection_merges( + plans: &[SelectionPlan], + selections: &mut [WorkbookSelection], + merged_ranges: &[String], + remaining_cells: &mut usize, +) { + let parsed_merges = merged_ranges + .iter() + .filter_map(|reference| parse_range_reference(reference).map(|bounds| (reference, bounds))) + .collect::>(); + for plan in plans { + let selection = &mut selections[plan.output_index]; + let intersecting = parsed_merges + .iter() + .filter(|(_, bounds)| bounds_intersect(plan.bounds, *bounds)) + .copied() + .collect::>(); + selection.merged_ranges = intersecting + .iter() + .map(|(reference, _)| (*reference).clone()) + .collect(); + let mut existing = selection + .cells + .iter() + .map(|cell| (cell.row, cell.column)) + .collect::>(); + for (reference, merge_bounds) in intersecting { + let anchor = format!( + "{}{}", + column_name(merge_bounds.start_column), + merge_bounds.start_row + ); + for cell in &mut selection.cells { + if contains(merge_bounds, cell.row, cell.column) && cell.merge.is_none() { + cell.merge = Some(merge_membership( + reference, + &anchor, + merge_bounds, + cell.row, + cell.column, + )); + } + } + let intersection = intersect_bounds(plan.bounds, merge_bounds); + 'rows: for row in intersection.start_row..=intersection.end_row { + for column in intersection.start_column..=intersection.end_column { + if existing.contains(&(row, column)) { + continue; + } + if *remaining_cells == 0 { + selection.truncated = true; + break 'rows; + } + selection.cells.push(blank_merge_cell( + row, + column, + merge_membership(reference, &anchor, merge_bounds, row, column), + )); + existing.insert((row, column)); + *remaining_cells -= 1; + } + } + } + selection.cells.sort_by_key(|cell| (cell.row, cell.column)); + } +} + +fn merge_membership( + range: &str, + anchor: &str, + bounds: SelectionBounds, + row: u32, + column: u16, +) -> WorkbookMergeMembership { + WorkbookMergeMembership { + range: range.to_owned(), + anchor: anchor.to_owned(), + role: if row == bounds.start_row && column == bounds.start_column { + WorkbookMergeRole::Anchor + } else { + WorkbookMergeRole::Covered + }, + } +} + +fn blank_merge_cell(row: u32, column: u16, merge: WorkbookMergeMembership) -> WorkbookCell { + WorkbookCell { + reference: format!("{}{row}", column_name(column)), + row, + column, + value_type: CellValueType::Blank, + value: String::new(), + display: String::new(), + formula: None, + formula_kind: None, + formula_reference: None, + shared_formula_index: None, + rich_text: false, + font_strike: None, + font_color: None, + rich_text_runs: Vec::new(), + merge: Some(merge), + style_index: None, + number_format: None, + } +} + +fn parse_range_reference(reference: &str) -> Option { + let (start, end) = reference + .split_once(':') + .map_or((reference, reference), |parts| parts); + let (start_row, start_column) = parse_cell_reference(start)?; + let (end_row, end_column) = parse_cell_reference(end)?; + (start_row <= end_row && start_column <= end_column).then_some(SelectionBounds { + start_row, + start_column, + end_row, + end_column, + }) +} + +fn bounds_intersect(left: SelectionBounds, right: SelectionBounds) -> bool { + left.start_row <= right.end_row + && right.start_row <= left.end_row + && left.start_column <= right.end_column + && right.start_column <= left.end_column +} + +fn intersect_bounds(left: SelectionBounds, right: SelectionBounds) -> SelectionBounds { + SelectionBounds { + start_row: left.start_row.max(right.start_row), + start_column: left.start_column.max(right.start_column), + end_row: left.end_row.min(right.end_row), + end_column: left.end_column.min(right.end_column), + } +} + fn capture_formula_metadata( cell: &mut CellBuilder, element: &BytesStart<'_>, @@ -2015,12 +3385,19 @@ fn finish_cell( ReadError::InvalidXlsx("worksheet cell has an invalid or missing reference".to_owned()) })?; let number_format = styles.number_format(cell.style_index); + let font = styles.font(cell.style_index); let raw = if cell.has_inline { cell.inline } else { cell.value }; let mut rich_text = cell.rich_text; + let mut rich_text_runs = cell.rich_text_runs; + for run in &mut rich_text_runs { + if let Some(color) = &mut run.font_color { + color.resolved_rgb = resolve_color(color, &styles.theme); + } + } let (value_type, display) = match cell.cell_type.as_deref() { Some("s") => { let index = raw.parse::().map_err(|_| { @@ -2029,6 +3406,7 @@ fn finish_cell( match shared_strings.get(index) { Some(value) => { rich_text = value.rich; + rich_text_runs = value.runs.clone(); (CellValueType::String, value.text.clone()) } None => { @@ -2083,6 +3461,10 @@ fn finish_cell( .then_some(cell.shared_formula_index) .flatten(), rich_text, + font_strike: font.and_then(|font| font.strike), + font_color: font.and_then(|font| font.color.clone()), + rich_text_runs, + merge: None, style_index: cell.style_index, number_format, })) @@ -2162,6 +3544,11 @@ fn push_selection( sheet, range: format_bounds(bounds), bounds, + used_bounds: None, + merged_ranges: Vec::new(), + images: Vec::new(), + images_truncated: false, + overflow: WorkbookSelectionOverflow::default(), cells: Vec::new(), truncated: false, }); @@ -2336,16 +3723,19 @@ fn render_markdown_generated_update( fn render_markdown_manifest(output: &mut String, workbook: &WorkbookInfo) { output.push_str("\n"); - output.push_str("| # | Sheet | State | Declared dimension | Semantic bounds | Selected |\n"); - output.push_str("|---:|---|---|---|---|---|\n"); + output.push_str( + "| # | Sheet | State | Declared dimension | Semantic bounds | Pictures | Selected |\n", + ); + output.push_str("|---:|---|---|---|---|---:|---|\n"); for sheet in &workbook.sheets { output.push_str(&format!( - "| {} | {} | {:?} | {} | {} | {} |\n", + "| {} | {} | {:?} | {} | {} | {} | {} |\n", sheet.index + 1, markdown_escape(&sheet.name), sheet.state, sheet.declared_dimension.as_deref().unwrap_or(""), sheet.semantic_bounds.as_deref().unwrap_or(""), + sheet.pictures.len(), if sheet.selected { "yes" } else { "no" } )); } @@ -2364,6 +3754,31 @@ fn render_markdown_manifest(output: &mut String, workbook: &WorkbookInfo) { )); } } + if workbook + .sheets + .iter() + .any(|sheet| has_print_evidence(&sheet.print)) + { + output.push_str("\n## Print evidence\n\n"); + output.push_str( + "| Sheet | Print area | Print titles | Page setup | Row breaks | Column breaks |\n|---|---|---|---|---|---|\n", + ); + for sheet in workbook + .sheets + .iter() + .filter(|sheet| has_print_evidence(&sheet.print)) + { + output.push_str(&format!( + "| {} | {} | {} | {} | {} | {} |\n", + markdown_escape(&sheet.name), + markdown_escape(sheet.print.print_area.as_deref().unwrap_or("")), + markdown_escape(sheet.print.print_titles.as_deref().unwrap_or("")), + markdown_escape(&page_setup_summary(sheet.print.page_setup.as_ref())), + join_u32(&sheet.print.row_breaks), + join_u32(&sheet.print.column_breaks), + )); + } + } output.push_str("\n"); } @@ -2377,6 +3792,36 @@ fn render_markdown_selection(output: &mut String, selection: &WorkbookSelection) markdown_escape(&selection.sheet), selection.range )); + if let Some(used_bounds) = &selection.used_bounds { + output.push_str(&format!( + "> Used cells on the requested rows span `{used_bounds}`.\n" + )); + } + render_markdown_selection_overflow(output, &selection.overflow); + if !selection.merged_ranges.is_empty() { + output.push_str(&format!( + "> Intersecting merges: {}. Covered cells remain blank and point to their anchor in JSON.\n", + selection + .merged_ranges + .iter() + .map(|range| format!("`{range}`")) + .collect::>() + .join(", ") + )); + } + if !selection.images.is_empty() { + output.push_str(&format!( + "> Intersecting worksheet pictures: {}. Pixels are not OCR text; open the bounded `dataUri` in JSON when present.\n", + selection.images.len() + )); + } + if selection.used_bounds.is_some() + || !selection.overflow.is_empty() + || !selection.merged_ranges.is_empty() + || !selection.images.is_empty() + { + output.push('\n'); + } let rows = u64::from(selection.bounds.end_row - selection.bounds.start_row + 1); let columns = u64::from(selection.bounds.end_column - selection.bounds.start_column + 1); if rows.saturating_mul(columns) <= PREVIEW_GRID_MAX_AREA && columns <= 50 { @@ -2393,12 +3838,124 @@ fn render_markdown_selection(output: &mut String, selection: &WorkbookSelection) )); } } + render_markdown_text_formatting(output, selection); + render_markdown_pictures(output, selection); if selection.truncated { output.push_str("\n> Selection output was truncated by the cell limit.\n"); } + if selection.images_truncated { + output.push_str("\n> Some picture payloads or picture references were omitted by image limits; metadata for retained entries remains available.\n"); + } output.push_str(&format!("\n")); } +fn render_markdown_pictures(output: &mut String, selection: &WorkbookSelection) { + if selection.images.is_empty() { + return; + } + output.push_str("\n### Worksheet pictures\n\n"); + output.push_str("| Anchor | Media part | Content type | Bytes | SHA-256 | Payload |\n"); + output.push_str("|---|---|---|---:|---|---|\n"); + for picture in &selection.images { + let anchor = picture.to_cell.as_ref().map_or_else( + || picture.from_cell.clone(), + |to| format!("{}:{}", picture.from_cell, to), + ); + let payload = if picture.data_uri.is_some() { + "dataUri in JSON" + } else if picture.payload_truncated { + "metadata only (limit)" + } else { + "metadata only" + }; + output.push_str(&format!( + "| {} | {} | {} | {} | `{}` | {} |\n", + markdown_escape(&anchor), + markdown_escape(&picture.media_part), + markdown_escape(&picture.content_type), + picture.byte_size, + picture.sha256, + payload, + )); + } + output.push_str("\n> Picture bytes are evidence only; Opsail does not OCR them or treat them as worksheet cell text.\n"); +} + +fn render_markdown_selection_overflow(output: &mut String, overflow: &WorkbookSelectionOverflow) { + if let Some(left) = &overflow.left { + output.push_str(&format!( + "> Column overflow left: {} through {} ({} used cells).\n", + column_name(left.min_column), + column_name(left.max_column), + left.cell_count + )); + } + if let Some(right) = &overflow.right { + output.push_str(&format!( + "> Column overflow right: {} through {} ({} used cells).\n", + column_name(right.min_column), + column_name(right.max_column), + right.cell_count + )); + } + if let Some(above) = &overflow.above { + output.push_str(&format!( + "> Scanned row overflow above: {} through {} ({} used cells in the requested columns).\n", + above.min_row, above.max_row, above.cell_count + )); + } + if let Some(below) = &overflow.below { + output.push_str(&format!( + "> Scanned row overflow below: {} through {} ({} used cells in the requested columns).\n", + below.min_row, below.max_row, below.cell_count + )); + } +} + +fn has_print_evidence(print: &WorksheetPrintEvidence) -> bool { + print.print_area.is_some() + || print.print_titles.is_some() + || print.page_setup.is_some() + || print.print_options.is_some() + || !print.row_breaks.is_empty() + || !print.column_breaks.is_empty() + || print.header_footer +} + +fn page_setup_summary(page_setup: Option<&WorksheetPageSetup>) -> String { + let Some(page_setup) = page_setup else { + return String::new(); + }; + let mut fields = Vec::new(); + if let Some(value) = &page_setup.orientation { + fields.push(format!("orientation={value}")); + } + if let Some(value) = page_setup.paper_size { + fields.push(format!("paperSize={value}")); + } + if let Some(value) = page_setup.fit_to_page { + fields.push(format!("fitToPage={value}")); + } + if let Some(value) = page_setup.fit_to_width { + fields.push(format!("fitToWidth={value}")); + } + if let Some(value) = page_setup.fit_to_height { + fields.push(format!("fitToHeight={value}")); + } + if let Some(value) = page_setup.scale { + fields.push(format!("scale={value}")); + } + fields.join(", ") +} + +fn join_u32(values: &[u32]) -> String { + values + .iter() + .map(u32::to_string) + .collect::>() + .join(", ") +} + /// Merge generated workbook blocks into an agent-maintained Markdown mirror. /// Text outside generated blocks is preserved byte-for-byte. A malformed /// existing mirror is rejected instead of being overwritten. @@ -2561,9 +4118,10 @@ fn render_html_manifest(output: &mut String, workbook: &WorkbookInfo) { output.push_str("\n
    "); for sheet in &workbook.sheets { output.push_str(&format!( - "
  • {} ({:?})
    • ", + "
  • {} ({:?}; {} worksheet pictures)
    • ", html_escape(&sheet.name), - sheet.state + sheet.state, + sheet.pictures.len() )); } output.push_str("
\n\n"); @@ -2572,10 +4130,37 @@ fn render_html_manifest(output: &mut String, workbook: &WorkbookInfo) { fn render_html_selection(output: &mut String, selection: &WorkbookSelection) { let marker = selection_marker(&selection.requested); output.push_str(&format!( - "\n

{}!{}

", + "\n

{}!{}

", html_escape(&selection.sheet), html_escape(&selection.range) )); + if let Some(used_bounds) = &selection.used_bounds { + output.push_str(&format!( + "

Used cells on the requested rows span {}.

", + html_escape(used_bounds) + )); + } + if let Some(right) = &selection.overflow.right { + output.push_str(&format!( + "

Column overflow right: {} through {} ({} used cells).

", + column_name(right.min_column), + column_name(right.max_column), + right.cell_count + )); + } + if !selection.merged_ranges.is_empty() { + output.push_str(&format!( + "

Intersecting merges: {}.

", + html_escape(&selection.merged_ranges.join(", ")) + )); + } + if !selection.images.is_empty() { + output.push_str(&format!( + "

{} intersecting worksheet pictures. Pixel payloads are bounded data URIs in JSON and are not OCR text.

", + selection.images.len() + )); + } + output.push_str("
CellValueTypeFormula
"); for cell in &selection.cells { output.push_str(&format!( "", @@ -2585,11 +4170,115 @@ fn render_html_selection(output: &mut String, selection: &WorkbookSelection) { html_escape(cell.formula.as_deref().unwrap_or("")) )); } + output.push_str("
CellValueTypeFormula
{}{}{:?}{}
"); + let formatted: Vec<_> = selection + .cells + .iter() + .filter_map(formatting_summary) + .collect(); + if !formatted.is_empty() { + output.push_str("

Text formatting

    "); + for summary in formatted { + output.push_str(&format!("
  • {}
    • ", html_escape(&summary))); + } + output.push_str("
"); + } + if !selection.images.is_empty() { + output.push_str("

Worksheet pictures

    "); + for picture in &selection.images { + let anchor = picture.to_cell.as_ref().map_or_else( + || picture.from_cell.clone(), + |to| format!("{}:{}", picture.from_cell, to), + ); + let payload = if picture.data_uri.is_some() { + "dataUri in JSON" + } else if picture.payload_truncated { + "metadata only (limit)" + } else { + "metadata only" + }; + output.push_str(&format!( + "
  • {}: {} ({} bytes, {}, {})
    • ", + html_escape(&anchor), + html_escape(&picture.media_part), + picture.byte_size, + html_escape(&picture.content_type), + payload, + )); + } + output.push_str("
"); + } output.push_str(&format!( - "\n\n" + "\n\n" )); } +fn render_markdown_text_formatting(output: &mut String, selection: &WorkbookSelection) { + let formatted: Vec<_> = selection + .cells + .iter() + .filter_map(formatting_summary) + .collect(); + if formatted.is_empty() { + return; + } + output.push_str("\n### Text formatting\n\n"); + for summary in formatted { + output.push_str(&format!("- {}\n", markdown_escape(&summary))); + } +} + +fn formatting_summary(cell: &WorkbookCell) -> Option { + let mut details = Vec::new(); + if cell.font_strike == Some(true) { + details.push("cell strike=true".to_owned()); + } + if let Some(color) = &cell.font_color { + details.push(format!("cell color={}", color_summary(color))); + } + for (index, run) in cell.rich_text_runs.iter().enumerate() { + if run.strike == Some(true) || run.font_color.is_some() { + let mut run_details = Vec::new(); + if run.strike == Some(true) { + run_details.push("strike=true".to_owned()); + } + if let Some(color) = &run.font_color { + run_details.push(format!("color={}", color_summary(color))); + } + details.push(format!( + "run {} {:?}: {}", + index + 1, + run.text, + run_details.join(", ") + )); + } + } + (!details.is_empty()).then(|| format!("{}: {}", cell.reference, details.join("; "))) +} + +fn color_summary(color: &WorkbookFontColor) -> String { + let mut fields = Vec::new(); + if let Some(value) = &color.rgb { + fields.push(format!("rgb={value}")); + } + if let Some(value) = color.theme { + fields.push(format!("theme={value}")); + } + if let Some(value) = color.indexed { + fields.push(format!("indexed={value}")); + } + if let Some(value) = color.auto { + fields.push(format!("auto={value}")); + } + if let Some(value) = color.tint { + fields.push(format!("tint={value}")); + } + if let Some(value) = &color.resolved_rgb { + fields.push(format!("resolvedRgb={value}")); + } + fields.join(",") +} + fn markdown_escape(value: &str) -> String { value .replace('|', "\\|") @@ -2822,17 +4511,57 @@ fn normalize_workbook_target(target: &str) -> Result { } fn worksheet_relationship_path(path: &str) -> Result { + part_relationship_path(path) +} + +fn part_relationship_path(path: &str) -> Result { let (directory, file) = path .rsplit_once('/') - .ok_or_else(|| ReadError::InvalidXlsx("worksheet part path has no directory".to_owned()))?; + .ok_or_else(|| ReadError::InvalidXlsx("OOXML part path has no directory".to_owned()))?; if file.is_empty() || directory.is_empty() { return Err(ReadError::InvalidXlsx( - "worksheet part path is invalid".to_owned(), + "OOXML part path is invalid".to_owned(), )); } Ok(format!("{directory}/_rels/{file}.rels")) } +fn resolve_part_target(source_part: &str, target: &str) -> Result { + let mut components = if target.starts_with('/') { + Vec::new() + } else { + source_part + .rsplit_once('/') + .map(|(directory, _)| { + directory + .split('/') + .filter(|component| !component.is_empty()) + .map(str::to_owned) + .collect::>() + }) + .unwrap_or_default() + }; + for component in target.trim_start_matches('/').split('/') { + match component { + "" | "." => {} + ".." => { + if components.pop().is_none() { + return Err(ReadError::InvalidXlsx( + "OOXML relationship target escapes the package root".to_owned(), + )); + } + } + component => components.push(component.to_owned()), + } + } + if components.is_empty() { + return Err(ReadError::InvalidXlsx( + "OOXML relationship target is empty".to_owned(), + )); + } + Ok(components.join("/")) +} + fn is_worksheet_part(name: &str) -> bool { name.starts_with("xl/worksheets/") && name.ends_with(".xml") && !name.contains("/_rels/") } @@ -2947,4 +4676,36 @@ mod tests { assert!(is_date_format("[h]:mm:ss")); assert!(is_date_format("[$-409]yyyy-mm-dd")); } + + #[test] + fn parses_one_cell_picture_anchors_and_skips_charts() { + let xml = r#" + 511 + + + 00 + 39 + + "#; + let relationships = HashMap::from([( + "rIdImage".to_owned(), + PartRelationship { + relationship_type: + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/image" + .to_owned(), + target: "../media/image1.png".to_owned(), + external: false, + }, + )]); + + let pictures = + parse_drawing_pictures(xml, "xl/drawings/drawing1.xml", &relationships).unwrap(); + assert_eq!(pictures.len(), 1); + assert_eq!(pictures[0].from.row, Some(11)); + assert_eq!(pictures[0].from.column, Some(5)); + assert!(pictures[0].to.is_none()); + assert_eq!(pictures[0].media_part, "xl/media/image1.png"); + } } diff --git a/crates/opsail-read/tests/xlsx.rs b/crates/opsail-read/tests/xlsx.rs index 594b208..5eb25bc 100644 --- a/crates/opsail-read/tests/xlsx.rs +++ b/crates/opsail-read/tests/xlsx.rs @@ -3,7 +3,8 @@ use std::path::Path; use opsail_read::{ CellValueType, FormulaKind, ReadArtifact, ReadError, ReadOptions, ReadSource, SheetState, - WorkbookPartRevision, WorkbookRevision, WorkbookSession, merge_markdown_mirror, read_artifact, + WorkbookMergeRole, WorkbookPartRevision, WorkbookRevision, WorkbookSession, + merge_markdown_mirror, read_artifact, }; use tempfile::tempdir; use zip::ZipWriter; @@ -30,6 +31,8 @@ const WORKBOOK: &str = r#" #REF! + 'Main'!$A$1:$Q$20 + 'Main'!$1:$2 "#; @@ -74,8 +77,11 @@ const POSITIVE_SHEET: &str = r#" - + + + + @@ -106,6 +112,17 @@ const ADVERSARIAL_SHEET: &str = r#" "#; +const TINY_PNG: &[u8] = &[ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x04, 0x00, 0x00, 0x00, 0xb5, 0x1c, 0x0c, + 0x02, 0x00, 0x00, 0x00, 0x0b, 0x49, 0x44, 0x41, 0x54, 0x78, 0xda, 0x63, 0x64, 0xf8, 0x0f, 0x00, + 0x01, 0x05, 0x01, 0x01, 0x27, 0x18, 0xe3, 0x66, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, + 0xae, 0x42, 0x60, 0x82, +]; + +const TINY_PNG_BASE64: &str = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="; + fn write_fixture(path: &Path, first_sheet: &str) { write_fixture_parts(path, first_sheet, SHARED_STRINGS); } @@ -119,6 +136,22 @@ fn write_fixture_parts_with_styles( first_sheet: &str, shared_strings: &str, styles: &str, +) { + write_fixture_parts_with_styles_and_theme( + path, + first_sheet, + shared_strings, + styles, + "", + ); +} + +fn write_fixture_parts_with_styles_and_theme( + path: &Path, + first_sheet: &str, + shared_strings: &str, + styles: &str, + theme: &str, ) { let file = std::fs::File::create(path).unwrap(); let mut zip = ZipWriter::new(file); @@ -132,7 +165,7 @@ fn write_fixture_parts_with_styles( ("xl/worksheets/sheet1.xml", first_sheet), ("xl/worksheets/_rels/sheet1.xml.rels", SHEET_RELATIONSHIPS), ("xl/worksheets/sheet2.xml", HIDDEN_SHEET), - ("xl/theme/theme1.xml", ""), + ("xl/theme/theme1.xml", theme), ("xl/drawings/drawing1.xml", ""), ("xl/charts/chart1.xml", ""), ("xl/media/image1.png", "fixture"), @@ -146,6 +179,416 @@ fn write_fixture_parts_with_styles( zip.finish().unwrap(); } +fn write_picture_fixture(path: &Path, media: &[u8]) { + let content_types = CONTENT_TYPES.replace( + r#""#, + r#" + "#, + ); + let sheet = r#" + + "#; + let drawing = r#" + + 0000 + 3090 + + + + + + "#; + let drawing_relationships = r#" + + "#; + let file = std::fs::File::create(path).unwrap(); + let mut zip = ZipWriter::new(file); + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + for (name, content) in [ + ("[Content_Types].xml", content_types.as_bytes()), + ("xl/workbook.xml", WORKBOOK.as_bytes()), + ("xl/_rels/workbook.xml.rels", RELATIONSHIPS.as_bytes()), + ("xl/styles.xml", STYLES.as_bytes()), + ("xl/sharedStrings.xml", SHARED_STRINGS.as_bytes()), + ("xl/worksheets/sheet1.xml", sheet.as_bytes()), + ( + "xl/worksheets/_rels/sheet1.xml.rels", + SHEET_RELATIONSHIPS.as_bytes(), + ), + ("xl/worksheets/sheet2.xml", HIDDEN_SHEET.as_bytes()), + ("xl/drawings/drawing1.xml", drawing.as_bytes()), + ( + "xl/drawings/_rels/drawing1.xml.rels", + drawing_relationships.as_bytes(), + ), + ("xl/media/image1.png", media), + ] { + zip.start_file(name, options).unwrap(); + zip.write_all(content).unwrap(); + } + zip.finish().unwrap(); +} + +#[tokio::test] +async fn returns_bounded_pixels_only_for_intersecting_worksheet_pictures() { + let directory = tempdir().unwrap(); + let path = directory.path().join("picture.xlsx"); + write_picture_fixture(&path, TINY_PNG); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:D10".to_owned(), "Main!E11:F12".to_owned()]; + + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!("expected workbook artifact"); + }; + + let inventory = &result.workbook.sheets[0].pictures; + assert_eq!(inventory.len(), 1); + assert_eq!(inventory[0].from_cell, "A1"); + assert_eq!(inventory[0].to_cell.as_deref(), Some("D10")); + assert_eq!(inventory[0].media_part, "xl/media/image1.png"); + assert_eq!(inventory[0].content_type, "image/png"); + assert_eq!(inventory[0].byte_size, TINY_PNG.len()); + assert_eq!( + inventory[0].sha256, + "431ced6916a2a21a156e38701afe55bbd7f88969fbbfc56d7fe099d47f265460" + ); + assert!(inventory[0].data_uri.is_none()); + + let hit = &result.workbook.selections[0]; + assert_eq!(hit.images.len(), 1); + assert_eq!(hit.images[0].from_row_index, 0); + assert_eq!(hit.images[0].from_column_index, 0); + assert_eq!(hit.images[0].to_row_index, Some(9)); + assert_eq!(hit.images[0].to_column_index, Some(3)); + assert_eq!( + hit.images[0].data_uri.as_deref(), + Some(format!("data:image/png;base64,{TINY_PNG_BASE64}").as_str()) + ); + assert!(!hit.images[0].payload_truncated); + assert!(!hit.images_truncated); + + let miss = &result.workbook.selections[1]; + assert!(miss.images.is_empty()); + assert!(!miss.images_truncated); + assert!( + result + .content + .contains("Intersecting worksheet pictures: 1") + ); + assert!(result.content.contains("dataUri in JSON")); + assert!(result.content.contains("does not OCR")); +} + +#[tokio::test] +async fn oversized_worksheet_picture_is_metadata_only() { + let directory = tempdir().unwrap(); + let path = directory.path().join("oversized-picture.xlsx"); + let media = vec![0_u8; 2 * 1024 * 1024 + 1]; + write_picture_fixture(&path, &media); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:D10".to_owned()]; + + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!("expected workbook artifact"); + }; + + let selection = &result.workbook.selections[0]; + assert_eq!(selection.images.len(), 1); + assert!(selection.images[0].data_uri.is_none()); + assert!(selection.images[0].payload_truncated); + assert!(selection.images_truncated); + assert!( + result + .warnings + .iter() + .any(|warning| warning.contains("2097152 bytes per image")) + ); + assert!(result.content.contains("metadata only (limit)")); +} + +#[tokio::test] +async fn publishes_cell_and_rich_run_strike_and_color_semantics() { + let directory = tempdir().unwrap(); + let path = directory.path().join("text-formatting.xlsx"); + let styles = r#" + + + + + "#; + let shared = r#" + deleted kept + "#; + let sheet = r#" + 0 + inline + plain + "#; + let theme = r#" + + + + "#; + write_fixture_parts_with_styles_and_theme(&path, sheet, shared, styles, theme); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:C1".to_owned()]; + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!() + }; + let cells = &result.workbook.selections[0].cells; + assert_eq!(cells[0].font_strike, Some(true)); + assert_eq!(cells[0].font_color.as_ref().unwrap().theme, Some(4)); + assert_eq!( + cells[0] + .font_color + .as_ref() + .unwrap() + .resolved_rgb + .as_deref(), + Some("8CB3D9") + ); + assert_eq!(cells[0].display, "deleted kept"); + assert!(cells[0].rich_text); + assert_eq!(cells[0].rich_text_runs[0].strike, Some(true)); + assert_eq!( + cells[0].rich_text_runs[0] + .font_color + .as_ref() + .unwrap() + .resolved_rgb + .as_deref(), + Some("123456") + ); + assert_eq!(cells[1].font_color.as_ref().unwrap().indexed, Some(10)); + assert_eq!( + cells[1] + .font_color + .as_ref() + .unwrap() + .resolved_rgb + .as_deref(), + Some("FF0000") + ); + assert_eq!(cells[1].rich_text_runs[0].strike, Some(false)); + assert_eq!( + cells[1].rich_text_runs[0] + .font_color + .as_ref() + .unwrap() + .theme, + Some(5) + ); + assert!( + cells[2].font_strike.is_none() + && cells[2].font_color.is_none() + && cells[2].rich_text_runs.is_empty() + ); + assert!(result.content.contains("### Text formatting")); + assert!(result.content.contains("run 1")); +} + +#[tokio::test] +async fn resolves_spreadsheet_theme_zero_one_and_respects_apply_font() { + let directory = tempdir().unwrap(); + let path = directory.path().join("theme-fonts.xlsx"); + let styles = r#" + + + + + + + + + + "#; + let sheet = r#" + light + dark + not-applied + direct + accent + automatic + "#; + let theme = r#" + + + + + + + + "#; + write_fixture_parts_with_styles_and_theme(&path, sheet, SHARED_STRINGS, styles, theme); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!A1:F1".to_owned()]; + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!() + }; + let color = |reference: &str| { + result.workbook.selections[0] + .cells + .iter() + .find(|cell| cell.reference == reference) + .unwrap() + .font_color + .as_ref() + .unwrap() + .resolved_rgb + .as_deref() + .unwrap() + }; + assert_eq!(color("A1"), "FFFFFF"); + assert_eq!(color("B1"), "000000"); + let not_applied = result.workbook.selections[0] + .cells + .iter() + .find(|cell| cell.reference == "C1") + .unwrap(); + assert!(not_applied.font_strike.is_none()); + assert!(not_applied.font_color.is_none()); + assert_eq!(color("D1"), "4F81BD"); + assert_eq!(color("E1"), "4BACC6"); + let automatic = result.workbook.selections[0] + .cells + .iter() + .find(|cell| cell.reference == "F1") + .unwrap() + .font_color + .as_ref() + .unwrap(); + assert_eq!(automatic.auto, Some(true)); + assert!(automatic.resolved_rgb.is_none()); +} + +#[tokio::test] +async fn publishes_merge_membership_for_horizontal_vertical_and_clipped_merges() { + let directory = tempdir().unwrap(); + let path = directory.path().join("merge-membership.xlsx"); + let sheet = r#" + + horizontal + vertical + wide + + + + "#; + write_fixture(&path, sheet); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!B1:M5".to_owned()]; + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!() + }; + let selection = &result.workbook.selections[0]; + assert_eq!(selection.merged_ranges, ["A1:C1", "D2:D4", "L5:Q5"]); + let cell = |reference: &str| { + selection + .cells + .iter() + .find(|cell| cell.reference == reference) + .unwrap() + }; + assert_eq!(cell("B1").merge.as_ref().unwrap().anchor, "A1"); + assert!(matches!( + cell("B1").merge.as_ref().unwrap().role, + WorkbookMergeRole::Covered + )); + assert!(matches!( + cell("D2").merge.as_ref().unwrap().role, + WorkbookMergeRole::Anchor + )); + assert_eq!(cell("D4").merge.as_ref().unwrap().anchor, "D2"); + assert!(matches!( + cell("M5").merge.as_ref().unwrap().role, + WorkbookMergeRole::Covered + )); + assert_eq!(cell("M5").merge.as_ref().unwrap().range, "L5:Q5"); + assert!(selection.cells.iter().all(|cell| cell.reference != "N5")); + assert!(result.content.contains("Intersecting merges")); + assert!(result.content.contains("`L5:Q5`")); +} + +#[tokio::test] +async fn scans_selected_rows_to_their_used_columns_and_still_reads_the_worksheet_tail() { + let directory = tempdir().unwrap(); + let path = directory.path().join("tail-and-overflow.xlsx"); + let sheet = r#" + + 1217 + 26 + + + + + + + + + "#; + write_fixture(&path, sheet); + let mut options = ReadOptions::default(); + options.spreadsheet.ranges = vec!["Main!B1:M1".to_owned()]; + let ReadArtifact::Workbook(result) = read_artifact(ReadSource::File(path), &options) + .await + .unwrap() + else { + panic!() + }; + let sheet = &result.workbook.sheets[0]; + assert!(!sheet.features.cell_data_complete); + assert!(sheet.features.tail_features_complete); + assert!(!sheet.features.complete); + assert_eq!(sheet.semantic_bounds.as_deref(), Some("A1:Q1")); + assert_eq!(sheet.merged_ranges, ["Y10:Z10"]); + assert_eq!(sheet.features.auto_filter.as_deref(), Some("A1:Q1")); + assert_eq!(sheet.print.page_setup.as_ref().unwrap().scale, Some(80)); + assert_eq!( + sheet.print.print_options.as_ref().unwrap().grid_lines, + Some(true) + ); + assert_eq!(sheet.print.row_breaks, [8]); + assert_eq!(sheet.print.column_breaks, [16]); + assert!(sheet.print.header_footer); + let selection = &result.workbook.selections[0]; + assert_eq!(selection.used_bounds.as_deref(), Some("A1:Q1")); + let left = selection.overflow.left.as_ref().unwrap(); + assert_eq!( + (left.min_column, left.max_column, left.cell_count), + (1, 1, 1) + ); + let right = selection.overflow.right.as_ref().unwrap(); + assert_eq!( + (right.min_column, right.max_column, right.cell_count), + (17, 17, 1) + ); + assert!(selection.overflow.below.is_none()); + assert!( + result + .content + .contains("Column overflow right: Q through Q") + ); + assert!(result.content.contains("'Main'!$A$1:$Q$20")); +} + #[tokio::test] async fn excludes_phonetic_runs_from_shared_string_display_text() { let directory = tempdir().unwrap(); @@ -261,6 +704,25 @@ fn revision_maps_changed_worksheet_relationships_back_to_their_sheet() { assert!(!diff.requires_full_refresh); } +#[test] +fn revision_treats_drawing_and_media_changes_as_dependency_wide() { + let revision = |crc32: &str| WorkbookRevision { + id: crc32.to_owned(), + compressed_bytes: 10, + expanded_bytes: 20, + parts: vec![WorkbookPartRevision { + name: "xl/media/image1.png".to_owned(), + crc32: crc32.to_owned(), + compressed_bytes: 10, + expanded_bytes: 20, + }], + }; + + let diff = revision("11111111").diff(&revision("22222222")); + assert!(diff.requires_full_refresh); + assert!(diff.changed_worksheet_parts.is_empty()); +} + fn large_tail_sheet(first_value: u32) -> String { let mut worksheet = format!( r#" @@ -307,8 +769,29 @@ async fn reads_values_formulas_dates_merges_and_visibility() { assert_eq!(result.workbook.sheets[0].hidden_rows, 1); assert_eq!(result.workbook.sheets[0].hidden_columns, 1); assert_eq!(result.workbook.sheets[0].merged_ranges, ["A4:B4"]); + assert!(result.workbook.sheets[0].features.cell_data_complete); + assert!(result.workbook.sheets[0].features.tail_features_complete); + let print = &result.workbook.sheets[0].print; + assert_eq!(print.print_area.as_deref(), Some("'Main'!$A$1:$Q$20")); + assert_eq!(print.print_titles.as_deref(), Some("'Main'!$1:$2")); + assert_eq!(print.page_setup.as_ref().unwrap().fit_to_width, Some(1)); + assert_eq!(print.print_options.as_ref().unwrap().headings, Some(true)); + assert_eq!(print.row_breaks, [3]); + assert_eq!(print.column_breaks, [2]); let selection = &result.workbook.selections[0]; - assert_eq!(selection.cells.len(), 8); + assert_eq!(selection.merged_ranges, ["A4:B4"]); + assert_eq!(selection.cells.len(), 9); + let covered = selection + .cells + .iter() + .find(|cell| cell.reference == "B4") + .unwrap(); + assert!(matches!(covered.value_type, CellValueType::Blank)); + assert!(matches!( + covered.merge.as_ref().unwrap().role, + WorkbookMergeRole::Covered + )); + assert_eq!(covered.merge.as_ref().unwrap().anchor, "A4"); assert_eq!( selection .cells @@ -530,7 +1013,7 @@ async fn does_not_allocate_from_a_declared_xfd_dimension() { } #[tokio::test] -async fn targeted_streaming_avoids_at_least_eighty_percent_of_expanded_bytes() { +async fn targeted_streaming_skips_later_cell_bodies_but_reaches_the_tail() { let directory = tempdir().unwrap(); let path = directory.path().join("large-tail.xlsx"); let worksheet = large_tail_sheet(1); @@ -551,10 +1034,10 @@ async fn targeted_streaming_avoids_at_least_eighty_percent_of_expanded_bytes() { panic!("expected workbook artifact"); }; - let full_bytes = full.workbook.statistics.expanded_bytes_read as f64; - let targeted_bytes = targeted.workbook.statistics.expanded_bytes_read as f64; - let saved = 1.0 - targeted_bytes / full_bytes; - assert!(saved >= 0.8, "expanded byte saving was {saved:.3}"); + assert_eq!(full.workbook.statistics.cell_elements, 20_001); + assert_eq!(targeted.workbook.statistics.cell_elements, 1); + assert!(!targeted.workbook.sheets[0].features.cell_data_complete); + assert!(targeted.workbook.sheets[0].features.tail_features_complete); assert!(!targeted.workbook.sheets[0].features.complete); assert!(full.workbook.features.inventory_complete); } @@ -583,7 +1066,13 @@ fn session_refreshes_human_xlsx_edits_and_preserves_agent_markdown() { assert!(refresh.metrics.changed); assert!(!refresh.metrics.full_refresh); assert_eq!(refresh.diff.changed_parts, ["xl/worksheets/sheet1.xml"]); - assert!(refresh.metrics.expanded_bytes_read * 5 <= cold_bytes); + assert!(refresh.metrics.expanded_bytes_read <= cold_bytes); + assert_eq!(refresh.result.workbook.statistics.cell_elements, 24); + assert!( + refresh.result.workbook.sheets[0] + .features + .tail_features_complete + ); assert_eq!(changed.value, "2"); assert!(mirror.contains("Keep this analysis across human edits.")); assert!(mirror.contains("| 1 | 2 |")); @@ -600,7 +1089,7 @@ fn incremental_refresh_counts_unchanged_cached_cells_against_the_global_limit() write_fixture(&path, POSITIVE_SHEET); let mut options = ReadOptions::default(); options.spreadsheet.ranges = vec!["Main!A1:D4".to_owned(), "'Hidden Data'!A1".to_owned()]; - options.spreadsheet.max_cells = 9; + options.spreadsheet.max_cells = 10; let mut session = WorkbookSession::open(path.clone(), options).unwrap(); assert_eq!( session @@ -610,7 +1099,7 @@ fn incremental_refresh_counts_unchanged_cached_cells_against_the_global_limit() .iter() .map(|selection| selection.cells.len()) .sum::(), - 9 + 10 ); let changed_sheet = POSITIVE_SHEET.replace( @@ -628,8 +1117,8 @@ fn incremental_refresh_counts_unchanged_cached_cells_against_the_global_limit() .sum::(); assert!(!refresh.metrics.full_refresh); - assert_eq!(returned, 9); - assert_eq!(refresh.result.workbook.statistics.returned_cells, 9); + assert_eq!(returned, 10); + assert_eq!(refresh.result.workbook.statistics.returned_cells, 10); assert!(refresh.result.workbook.selections[0].truncated); assert_eq!(refresh.result.workbook.selections[1].cells.len(), 1); } diff --git a/crates/opsail/tests/xlsx_cli.rs b/crates/opsail/tests/xlsx_cli.rs index a60e8fe..599c5f4 100644 --- a/crates/opsail/tests/xlsx_cli.rs +++ b/crates/opsail/tests/xlsx_cli.rs @@ -42,6 +42,95 @@ fn write_xlsx_with_sheet(path: &Path, sheet: &str) { archive.finish().unwrap(); } +fn write_picture_xlsx(path: &Path) { + const PNG: &[u8] = &[ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, + 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x04, 0x00, 0x00, 0x00, 0xb5, + 0x1c, 0x0c, 0x02, 0x00, 0x00, 0x00, 0x0b, 0x49, 0x44, 0x41, 0x54, 0x78, 0xda, 0x63, 0x64, + 0xf8, 0x0f, 0x00, 0x01, 0x05, 0x01, 0x01, 0x27, 0x18, 0xe3, 0x66, 0x00, 0x00, 0x00, 0x00, + 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]; + let parts: [(&str, &[u8]); 7] = [ + ( + "[Content_Types].xml", + br#""#, + ), + ( + "xl/workbook.xml", + br#""#, + ), + ( + "xl/_rels/workbook.xml.rels", + br#""#, + ), + ( + "xl/worksheets/sheet1.xml", + br#""#, + ), + ( + "xl/worksheets/_rels/sheet1.xml.rels", + br#""#, + ), + ( + "xl/drawings/drawing1.xml", + br#"0039"#, + ), + ( + "xl/drawings/_rels/drawing1.xml.rels", + br#""#, + ), + ]; + let file = std::fs::File::create(path).unwrap(); + let mut archive = ZipWriter::new(file); + for (name, bytes) in parts { + archive + .start_file(name, SimpleFileOptions::default()) + .unwrap(); + archive.write_all(bytes).unwrap(); + } + archive + .start_file("xl/media/image1.png", SimpleFileOptions::default()) + .unwrap(); + archive.write_all(PNG).unwrap(); + archive.finish().unwrap(); +} + +#[test] +fn read_cli_returns_bounded_picture_pixels_for_an_intersecting_range() { + let directory = tempdir().unwrap(); + let path = directory.path().join("picture.xlsx"); + write_picture_xlsx(&path); + let assert = Command::cargo_bin("opsail") + .unwrap() + .args([ + "read", + path.to_str().unwrap(), + "--range", + "Data!A1:D10", + "--format", + "json", + ]) + .assert() + .success(); + let output = assert.get_output(); + let result: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + + assert!(output.stdout.len() < 16 * 1024 * 1024); + let inventory = &result["workbook"]["sheets"][0]["pictures"][0]; + assert_eq!(inventory["fromCell"], "A1"); + assert_eq!(inventory["toCell"], "D10"); + assert!(inventory.get("dataUri").is_none()); + let selected = &result["workbook"]["selections"][0]["images"][0]; + assert_eq!(selected["contentType"], "image/png"); + assert!( + selected["dataUri"] + .as_str() + .unwrap() + .starts_with("data:image/png;base64,iVBORw0KGgo") + ); + assert_eq!(selected["payloadTruncated"], false); +} + #[test] fn read_cli_truncates_cells_to_the_serialized_output_budget() { let directory = tempdir().unwrap(); @@ -105,7 +194,7 @@ fn read_cli_emits_a_sparse_workbook_artifact() { ]) .assert() .success() - .stderr(predicate::str::contains("targeted worksheet scan stopped")); + .stderr(predicate::str::contains("skipped cell bodies")); let result: serde_json::Value = serde_json::from_slice(&assert.get_output().stdout).unwrap(); assert_eq!(result["artifactKind"], "workbook"); @@ -116,6 +205,19 @@ fn read_cli_emits_a_sparse_workbook_artifact() { result["workbook"]["sheets"][0]["semanticBoundsComplete"], false ); + assert_eq!( + result["workbook"]["sheets"][0]["features"]["cellDataComplete"], + false + ); + assert_eq!( + result["workbook"]["sheets"][0]["features"]["tailFeaturesComplete"], + true + ); + assert_eq!(result["workbook"]["selections"][0]["usedBounds"], "A1:A1"); + assert_eq!( + result["workbook"]["selections"][0]["mergedRanges"], + json!([]) + ); assert_eq!( result["workbook"]["selections"][0]["cells"][0]["display"], "CLI value" diff --git a/packages/node/index.d.ts b/packages/node/index.d.ts index 9472c4b..d818142 100644 --- a/packages/node/index.d.ts +++ b/packages/node/index.d.ts @@ -150,11 +150,38 @@ export interface WorkbookSheet { semanticBoundsComplete: boolean; selected: boolean; mergedRanges: string[]; + /** Worksheet picture inventory. Payloads are published only on intersecting selections. */ + pictures: WorkbookPicture[]; hiddenRows: number; hiddenColumns: number; + print: WorksheetPrintEvidence; features: WorksheetFeatureInventory; } +export interface WorksheetPrintEvidence { + printArea?: string; + printTitles?: string; + pageSetup?: WorksheetPageSetup; + printOptions?: WorksheetPrintOptions; + rowBreaks: number[]; + columnBreaks: number[]; + headerFooter: boolean; +} + +export interface WorksheetPageSetup { + orientation?: string; + paperSize?: number; + fitToPage?: boolean; + fitToWidth?: number; + fitToHeight?: number; + scale?: number; +} + +export interface WorksheetPrintOptions { + gridLines?: boolean; + headings?: boolean; +} + export interface WorkbookRevision { id: string; compressedBytes: number; @@ -194,6 +221,8 @@ export interface WorkbookHyperlink { export interface WorksheetFeatureInventory { scanned: boolean; + cellDataComplete: boolean; + tailFeaturesComplete: boolean; complete: boolean; featureReferencesTruncated: boolean; formulaCells: number; @@ -233,10 +262,55 @@ export interface WorkbookSelection { endRow: number; endColumn: number; }; + usedBounds?: string; + mergedRanges: string[]; + /** Pictures whose DrawingML anchors intersect the requested rectangle. */ + images: WorkbookPicture[]; + /** Independent of cell `truncated`; true when image payload/reference caps were hit. */ + imagesTruncated: boolean; + overflow?: WorkbookSelectionOverflow; cells: WorkbookCell[]; truncated: boolean; } +export interface WorkbookPicture { + sheet: string; + fromCell: string; + toCell?: string; + /** Zero-based OOXML drawing marker index. */ + fromRowIndex: number; + /** Zero-based OOXML drawing marker index. */ + fromColumnIndex: number; + toRowIndex?: number; + toColumnIndex?: number; + mediaPart: string; + contentType: string; + byteSize: number; + sha256: string; + /** Bounded image payload; absent from sheet inventory and when limits are hit. */ + dataUri?: string; + payloadTruncated: boolean; +} + +export interface WorkbookSelectionOverflow { + left?: WorkbookColumnOverflow; + right?: WorkbookColumnOverflow; + above?: WorkbookRowOverflow; + below?: WorkbookRowOverflow; +} + +export interface WorkbookColumnOverflow { + minColumn: number; + maxColumn: number; + cellCount: number; +} + +export interface WorkbookRowOverflow { + minRow: number; + maxRow: number; + cellCount: number; +} + export interface WorkbookCell { reference: string; row: number; @@ -249,10 +323,35 @@ export interface WorkbookCell { formulaReference?: string; sharedFormulaIndex?: number; richText: boolean; + fontStrike?: boolean; + fontColor?: WorkbookFontColor; + richTextRuns?: WorkbookRichTextRun[]; + merge?: WorkbookMergeMembership; styleIndex?: number; numberFormat?: string; } +export interface WorkbookMergeMembership { + range: string; + anchor: string; + role: "anchor" | "covered"; +} + +export interface WorkbookFontColor { + rgb?: string; + theme?: number; + indexed?: number; + auto?: boolean; + tint?: number; + resolvedRgb?: string; +} + +export interface WorkbookRichTextRun { + text: string; + strike?: boolean; + fontColor?: WorkbookFontColor; +} + export interface WorkbookStatistics { archiveEntries: number; expandedBytesRead: number; diff --git a/packages/node/src/client.js b/packages/node/src/client.js index 2539b88..5abfa77 100644 --- a/packages/node/src/client.js +++ b/packages/node/src/client.js @@ -619,12 +619,47 @@ function isWorkbookSheet(value) { typeof value.selected === "boolean" && Array.isArray(value.mergedRanges) && value.mergedRanges.every((range) => typeof range === "string") && + Array.isArray(value.pictures) && + value.pictures.every(isWorkbookPicture) && isNonNegativeSafeInteger(value.hiddenRows) && isNonNegativeSafeInteger(value.hiddenColumns) && + isWorksheetPrintEvidence(value.print) && isWorksheetFeatureInventory(value.features) ); } +function isWorksheetPrintEvidence(value) { + return ( + isRecord(value) && + isOptionalString(value.printArea) && + isOptionalString(value.printTitles) && + (value.pageSetup === undefined || isWorksheetPageSetup(value.pageSetup)) && + (value.printOptions === undefined || + (isRecord(value.printOptions) && + (value.printOptions.gridLines === undefined || + typeof value.printOptions.gridLines === "boolean") && + (value.printOptions.headings === undefined || + typeof value.printOptions.headings === "boolean"))) && + Array.isArray(value.rowBreaks) && + value.rowBreaks.every(isNonNegativeSafeInteger) && + Array.isArray(value.columnBreaks) && + value.columnBreaks.every(isNonNegativeSafeInteger) && + typeof value.headerFooter === "boolean" + ); +} + +function isWorksheetPageSetup(value) { + return ( + isRecord(value) && + isOptionalString(value.orientation) && + ["paperSize", "fitToWidth", "fitToHeight", "scale"].every( + (field) => + value[field] === undefined || isNonNegativeSafeInteger(value[field]), + ) && + (value.fitToPage === undefined || typeof value.fitToPage === "boolean") + ); +} + function isDefinedName(value) { return ( isRecord(value) && @@ -643,12 +678,78 @@ function isWorkbookSelection(value) { typeof value.sheet === "string" && typeof value.range === "string" && isSelectionBounds(value.bounds) && + isOptionalString(value.usedBounds) && + Array.isArray(value.mergedRanges) && + value.mergedRanges.every((range) => typeof range === "string") && + Array.isArray(value.images) && + value.images.every(isWorkbookPicture) && + typeof value.imagesTruncated === "boolean" && + (value.overflow === undefined || isWorkbookSelectionOverflow(value.overflow)) && Array.isArray(value.cells) && value.cells.every(isWorkbookCell) && typeof value.truncated === "boolean" ); } +function isWorkbookPicture(value) { + return ( + isRecord(value) && + typeof value.sheet === "string" && + typeof value.fromCell === "string" && + isOptionalString(value.toCell) && + isNonNegativeSafeInteger(value.fromRowIndex) && + isNonNegativeSafeInteger(value.fromColumnIndex) && + (value.toRowIndex === undefined || + isNonNegativeSafeInteger(value.toRowIndex)) && + (value.toColumnIndex === undefined || + isNonNegativeSafeInteger(value.toColumnIndex)) && + typeof value.mediaPart === "string" && + typeof value.contentType === "string" && + isNonNegativeSafeInteger(value.byteSize) && + typeof value.sha256 === "string" && + /^[0-9a-f]{64}$/.test(value.sha256) && + (value.dataUri === undefined || + (typeof value.dataUri === "string" && + value.dataUri.startsWith(`data:${value.contentType};base64,`))) && + typeof value.payloadTruncated === "boolean" + ); +} + +function isWorkbookSelectionOverflow(value) { + return ( + isRecord(value) && + ["left", "right"].every( + (field) => + value[field] === undefined || isWorkbookColumnOverflow(value[field]), + ) && + ["above", "below"].every( + (field) => value[field] === undefined || isWorkbookRowOverflow(value[field]), + ) + ); +} + +function isWorkbookColumnOverflow(value) { + return ( + isRecord(value) && + Number.isSafeInteger(value.minColumn) && + value.minColumn > 0 && + Number.isSafeInteger(value.maxColumn) && + value.maxColumn >= value.minColumn && + isNonNegativeSafeInteger(value.cellCount) + ); +} + +function isWorkbookRowOverflow(value) { + return ( + isRecord(value) && + Number.isSafeInteger(value.minRow) && + value.minRow > 0 && + Number.isSafeInteger(value.maxRow) && + value.maxRow >= value.minRow && + isNonNegativeSafeInteger(value.cellCount) + ); +} + function isSelectionBounds(value) { return ( isRecord(value) && @@ -685,12 +786,49 @@ function isWorkbookCell(value) { (value.sharedFormulaIndex === undefined || isNonNegativeSafeInteger(value.sharedFormulaIndex)) && typeof value.richText === "boolean" && + (value.fontStrike === undefined || typeof value.fontStrike === "boolean") && + (value.fontColor === undefined || isWorkbookFontColor(value.fontColor)) && + (value.richTextRuns === undefined || + (Array.isArray(value.richTextRuns) && + value.richTextRuns.every(isWorkbookRichTextRun))) && + (value.merge === undefined || isWorkbookMergeMembership(value.merge)) && (value.styleIndex === undefined || isNonNegativeSafeInteger(value.styleIndex)) && isOptionalString(value.numberFormat) ); } +function isWorkbookMergeMembership(value) { + return ( + isRecord(value) && + typeof value.range === "string" && + typeof value.anchor === "string" && + (value.role === "anchor" || value.role === "covered") + ); +} + +function isWorkbookFontColor(value) { + return ( + isRecord(value) && + isOptionalString(value.rgb) && + (value.theme === undefined || isNonNegativeSafeInteger(value.theme)) && + (value.indexed === undefined || isNonNegativeSafeInteger(value.indexed)) && + (value.auto === undefined || typeof value.auto === "boolean") && + (value.tint === undefined || + (typeof value.tint === "number" && Number.isFinite(value.tint))) && + isOptionalString(value.resolvedRgb) + ); +} + +function isWorkbookRichTextRun(value) { + return ( + isRecord(value) && + typeof value.text === "string" && + (value.strike === undefined || typeof value.strike === "boolean") && + (value.fontColor === undefined || isWorkbookFontColor(value.fontColor)) + ); +} + function isWorkbookRevision(value) { return ( isRecord(value) && @@ -736,6 +874,8 @@ function isWorksheetFeatureInventory(value) { return ( isRecord(value) && typeof value.scanned === "boolean" && + typeof value.cellDataComplete === "boolean" && + typeof value.tailFeaturesComplete === "boolean" && typeof value.complete === "boolean" && typeof value.featureReferencesTruncated === "boolean" && [ diff --git a/packages/node/test/protocol.test.js b/packages/node/test/protocol.test.js index a99c687..5b7c66f 100644 --- a/packages/node/test/protocol.test.js +++ b/packages/node/test/protocol.test.js @@ -89,10 +89,43 @@ function validWorkbookResult() { semanticBoundsComplete: false, selected: true, mergedRanges: [], + pictures: [ + { + sheet: "Data", + fromCell: "A1", + toCell: "D10", + fromRowIndex: 0, + fromColumnIndex: 0, + toRowIndex: 9, + toColumnIndex: 3, + mediaPart: "xl/media/image1.png", + contentType: "image/png", + byteSize: 68, + sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + payloadTruncated: false, + }, + ], hiddenRows: 0, hiddenColumns: 0, + print: { + printArea: "'Data'!$A$1:$Q$20", + printTitles: "'Data'!$1:$2", + pageSetup: { + orientation: "landscape", + paperSize: 9, + fitToPage: true, + fitToWidth: 1, + fitToHeight: 0, + }, + printOptions: { gridLines: false, headings: true }, + rowBreaks: [20], + columnBreaks: [17], + headerFooter: true, + }, features: { scanned: true, + cellDataComplete: false, + tailFeaturesComplete: true, complete: false, featureReferencesTruncated: false, formulaCells: 0, @@ -127,6 +160,29 @@ function validWorkbookResult() { endRow: 2, endColumn: 2, }, + usedBounds: "A1:Q2", + mergedRanges: ["A1:C1"], + images: [ + { + sheet: "Data", + fromCell: "A1", + toCell: "D10", + fromRowIndex: 0, + fromColumnIndex: 0, + toRowIndex: 9, + toColumnIndex: 3, + mediaPart: "xl/media/image1.png", + contentType: "image/png", + byteSize: 68, + sha256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + dataUri: "data:image/png;base64,iVBORw0KGgo=", + payloadTruncated: false, + }, + ], + imagesTruncated: false, + overflow: { + right: { minColumn: 17, maxColumn: 17, cellCount: 1 }, + }, cells: [ { reference: "A1", @@ -135,7 +191,13 @@ function validWorkbookResult() { valueType: "string", value: "value", display: "value", - richText: false, + richText: true, + fontStrike: true, + fontColor: { theme: 4, tint: 0.5, resolvedRgb: "8CB3D9" }, + richTextRuns: [ + { text: "value", strike: true, fontColor: { rgb: "FF123456", resolvedRgb: "123456" } }, + ], + merge: { range: "A1:C1", anchor: "A1", role: "anchor" }, }, ], truncated: false, @@ -231,6 +293,30 @@ test("machine workbook validation rejects inconsistent structured fields", () => (result) => { result.workbook.selections[0].cells[0].richText = 1; }, + (result) => { + result.workbook.selections[0].cells[0].fontColor.tint = "light"; + }, + (result) => { + result.workbook.selections[0].cells[0].richTextRuns[0].strike = 1; + }, + (result) => { + result.workbook.selections[0].overflow.right.maxColumn = 0; + }, + (result) => { + result.workbook.selections[0].cells[0].merge.role = "merged"; + }, + (result) => { + result.workbook.sheets[0].print.pageSetup.fitToWidth = -1; + }, + (result) => { + result.workbook.selections[0].images[0].sha256 = "not-a-hash"; + }, + (result) => { + result.workbook.selections[0].images[0].dataUri = "data:text/plain;base64,eA=="; + }, + (result) => { + result.workbook.selections[0].imagesTruncated = "no"; + }, ]) { const result = validWorkbookResult(); mutate(result); diff --git a/skills/opsail/SKILL.md b/skills/opsail/SKILL.md index 7e71ea7..90e0aa1 100644 --- a/skills/opsail/SKILL.md +++ b/skills/opsail/SKILL.md @@ -64,7 +64,12 @@ opsail read './book.xlsx' \ `--max-expanded-bytes` limits cumulative uncompressed OOXML bytes. `--no-formulas` omits formula expressions but preserves cached values. Formula values are never recalculated and may be stale. Treat `declaredDimension`, `semanticBounds`, `styleOnlyCells`, `truncated`, invalid defined names, -and warnings as separate evidence. +and warnings as separate evidence. Scanned sheets inventory worksheet pictures in metadata-only +`pictures`; intersecting selections publish `images` with anchors, media metadata, SHA-256, and a +bounded `dataUri` when available. Picture references are capped at 256 per sheet/selection; open +the URI directly or decode its base64 bytes. Image +`imagesTruncated`/`payloadTruncated` signals are independent from cell `truncated`; pictures are +not OCR text. For a repeated read, probe the content revision before rereading cell XML: @@ -75,7 +80,8 @@ opsail read './book.xlsx' --revision-only --property revision --format json Reuse an existing Markdown mirror when the revision is unchanged. When a worksheet part changed, reread the previously selected ranges in one invocation and replace only matching `opsail:xlsx-generated` blocks; keep agent-authored text outside those blocks. Workbook, -shared-string, style, or theme changes are dependency-wide and require a conservative full read. +shared-string, style, theme, drawing, or media changes are dependency-wide and require a +conservative full read. The CLI invocation itself is stateless; the high-efficiency `WorkbookSession` cache is an in-process `opsail-read` Rust API, so do not claim session-level timings for repeated CLI processes.