7.0: parallel XLSX/XLSB reading, Excel tables, own byte sources, managed inflate, and a public API cleanup

.github/workflows/benchmark-pr.yml

@@ -22,7 +22,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       benchmarks: ${{ steps.filter.outputs.benchmarks }}
-      run: ${{ steps.filter.outputs.benchmarks == 'true' && (github.base_ref == 'master' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) }}
+      # Opt-in on every base branch: the suite is ~40 runner-minutes, too much to repeat on each push.
+      run: ${{ steps.filter.outputs.benchmarks == 'true' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }}
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -66,7 +67,7 @@ jobs:
           cache-dependency-path: '**/*.csproj'
 
       - name: Build (Release)
-        run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release
+        run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release -p:RunAnalyzers=false
 
       - name: Check group coverage
         run: bash tests/ExcelReader.Benchmarks/check-benchmark-groups.sh
@@ -94,7 +95,7 @@ jobs:
         run: dotnet restore tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj
 
       - name: Build (Release)
-        run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release --no-restore
+        run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release --no-restore -p:RunAnalyzers=false
 
       - name: Run benchmarks
         run: |

.github/workflows/ci.yml

@@ -36,6 +36,7 @@ jobs:
             dotnet:
               - 'src/**'
               - 'tests/**'
+              - 'docs/snippets/**'
               - 'Directory.Build.props'
               - 'global.json'
               - 'ExcelReader.slnx'
@@ -64,6 +65,12 @@ jobs:
         os: [ubuntu-latest, windows-latest, macos-latest]
     env:
       RUN: ${{ needs.changes.outputs.dotnet }}
+      # The whole solution compiles the same on every OS, so Linux builds all of it and the others
+      # build only what the tests need.
+      BUILD_TARGET: ${{ matrix.os == 'ubuntu-latest' && 'ExcelReader.slnx' || 'tests/ExcelReader.Tests/ExcelReader.Tests.csproj' }}
+      # Analyzers are about two thirds of the compile and report the same on every OS, so only the
+      # Linux build runs them; every other build in CI skips them. Source generators still run.
+      BUILD_ARGS: ${{ matrix.os != 'ubuntu-latest' && '-p:RunAnalyzers=false' || '' }}
 
     steps:
       - name: Nothing to build
@@ -84,11 +91,11 @@ jobs:
 
       - name: Restore
         if: env.RUN == 'true'
-        run: dotnet restore ExcelReader.slnx
+        run: dotnet restore ${{ env.BUILD_TARGET }}
 
       - name: Build (Release)
         if: env.RUN == 'true'
-        run: dotnet build ExcelReader.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false
+        run: dotnet build ${{ env.BUILD_TARGET }} --configuration Release --no-restore -p:DeterministicSourcePaths=false ${{ env.BUILD_ARGS }}
 
       - name: Verify formatting (whitespace)
         if: env.RUN == 'true' && matrix.os == 'ubuntu-latest'
@@ -155,7 +162,7 @@ jobs:
           cache-dependency-path: '**/*.csproj'
 
       - name: Build (Release)
-        run: dotnet build tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configuration Release
+        run: dotnet build tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configuration Release -p:RunAnalyzers=false
 
       - name: Test (Release)
         run: dotnet test --project tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configuration Release --no-build
@@ -188,7 +195,7 @@ jobs:
       - name: Pack and verify the generator ships inside the nupkg
         shell: pwsh
         run: |
-          dotnet pack src/ExcelReader.Core/ExcelReader.Core.csproj -c Release -o ./artifacts
+          dotnet pack src/ExcelReader.Core/ExcelReader.Core.csproj -c Release -o ./artifacts -p:RunAnalyzers=false
           $pkg = Get-ChildItem ./artifacts/*.nupkg | Select-Object -First 1
           Add-Type -AssemblyName System.IO.Compression.FileSystem
           $zip = [System.IO.Compression.ZipFile]::OpenRead($pkg.FullName)

.github/workflows/codeql.yml

@@ -1,8 +1,9 @@
 name: CodeQL
 
 on:
+  # develop is covered by its pull request into master; a push trigger there would run everything twice.
   push:
-    branches: [ master, develop ]
+    branches: [ master ]
   pull_request:
     branches: [ master, develop ]
     types: [ opened, synchronize, reopened, ready_for_review ]
@@ -89,9 +90,14 @@ jobs:
           languages: ${{ matrix.language }}
           queries: security-extended
 
+      # Only the shipped projects: tests, benchmarks and the fuzz harness double the build for code
+      # no one runs against untrusted input.
       - name: Build (Release)
         if: matrix.language == 'csharp'
-        run: dotnet build ExcelReader.slnx --configuration Release
+        run: |
+          for project in src/*/*.csproj; do
+            dotnet build "$project" --configuration Release -p:RunAnalyzers=false
+          done
 
       - name: Perform CodeQL Analysis
         uses: github/codeql-action/analyze@v4

.github/workflows/docs.yml

@@ -0,0 +1,74 @@
+name: Docs -> GitHub Pages
+
+on:
+  push:
+    branches: [master]
+    paths: &docs-paths
+      - 'src/ExcelReader.Core/**'
+      - 'src/ExcelReader.Arrow/**'
+      - 'src/ExcelReader.Native/include/**'
+      - 'python/src/**'
+      - 'python/scripts/api_markdown.py'
+      - 'cpp/scripts/api_markdown.py'
+      - 'docs/**'
+      - '**.md'
+      - 'docfx.json'
+      - 'assets/**'
+      - '.github/workflows/docs.yml'
+  # Builds without publishing, so a broken page or a Doxygen @param that no longer matches its
+  # signature fails the pull request instead of the next publish.
+  pull_request:
+    paths: *docs-paths
+  workflow_dispatch:
+
+concurrency:
+  group: docs-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-dotnet@v6
+        with:
+          global-json-file: global.json
+
+      # The snippets themselves are compiled by ci.yml, which builds the whole solution.
+      - name: Check guide examples match docs/snippets
+        run: python3 docs/snippets/sync.py --check
+
+      # Importing excelreader loads the native library lazily, so this needs only the sources.
+      - name: Build Python API reference
+        run: python3 python/scripts/api_markdown.py
+
+      # Fails on any Doxygen documentation warning, such as an @param naming no parameter.
+      - name: Build C++ API reference
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes --no-install-recommends doxygen
+          python3 cpp/scripts/api_markdown.py
+
+      # Fails on a broken page link or an xref naming a member that no longer exists. Core loads the
+      # generator as an analyzer from its bin/Release output, so it is built first, or docfx warns
+      # FailedToResolveAnalyzer.
+      - name: Build site
+        run: |
+          dotnet build src/ExcelReader.Generator/ExcelReader.Generator.csproj --configuration Release -p:RunAnalyzers=false
+          dotnet tool install --global docfx --version 2.81.0
+          docfx docfx.json --warningsAsErrors
+
+      # keep_files leaves dev/bench (benchmark-pages.yml) in place; pages removed from the docs
+      # stay on the branch until deleted by hand.
+      - name: Publish to gh-pages
+        if: github.event_name != 'pull_request'
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs/_site
+          publish_branch: gh-pages
+          keep_files: true

.github/workflows/fuzz.yml

@@ -53,7 +53,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        target: [xlsx, xlsx-memory, xlsx-differential, xlsb, xlsb-memory, xlsb-differential, xls, encrypted, csv, csv-sniff, csv-parallel]
+        target: [xlsx, xlsx-memory, xlsx-differential, xlsx-pipeline, xlsb, xlsb-memory, xlsb-differential, xlsb-pipeline, xls, encrypted, csv, csv-sniff, csv-parallel, inflate]
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -76,7 +76,7 @@ jobs:
 
       - name: Publish harness
         run: |
-          dotnet publish tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release -o fuzz-out
+          dotnet publish tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release -o fuzz-out -p:RunAnalyzers=false
           chmod +x fuzz-out/ExcelReader.Fuzz
 
       - name: Generate seed corpus
@@ -110,6 +110,7 @@ jobs:
             xls)       dict=xls ;;
             encrypted) dict=encrypted ;;
             csv*)      dict=csv ;;
+            inflate)   dict=inflate ;;
           esac
           ./libfuzzer-dotnet \
             --target_path=fuzz-out/ExcelReader.Fuzz \
@@ -158,7 +159,7 @@ jobs:
 
       - name: Build
         if: env.RUN == 'true'
-        run: dotnet build tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release
+        run: dotnet build tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release -p:RunAnalyzers=false
 
       - name: Generate corpus
         if: env.RUN == 'true'

.github/workflows/native-bindings.yml

@@ -1,8 +1,9 @@
 name: Native bindings
 
 on:
+  # develop is covered by its pull request into master; a push trigger there would run everything twice.
   push:
-    branches: [master, develop]
+    branches: [master]
   pull_request:
     branches: [master, develop]
     types: [opened, synchronize, reopened, ready_for_review]
@@ -76,6 +77,8 @@ jobs:
 
       - name: Build the native library
         shell: bash
+        env:
+          RunAnalyzers: 'false'
         run: python python/scripts/build_native.py
 
       - name: Upload native library
@@ -145,13 +148,17 @@ jobs:
           EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }}
         run: cargo test --workspace --all-features --verbose
 
+      # The next two compile the crate again (default features, then the bench profile); that
+      # catches nothing platform-specific the test build above missed, so one OS is enough.
       - name: cargo build (default features)
+        if: runner.os == 'Linux'
         working-directory: rust
         env:
           EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }}
         run: cargo build --workspace --verbose
 
       - name: cargo bench (smoke)
+        if: runner.os == 'Linux'
         working-directory: rust
         env:
           EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }}

.github/workflows/release.yml

@@ -312,8 +312,6 @@ jobs:
             grep -q 'already exists' <<<"$out" || exit 1
           fi
 
-      # The workspace config points EXCELREADER_NATIVE_LIB_DIR at a local build that this job never makes;
-      # without it the package verification downloads the native library from the release, as users do.
       - name: Publish excelreader to crates.io
         working-directory: rust/excelreader
         env:

.gitignore

@@ -506,3 +506,8 @@ python/src/excelreader/_lib/*
 build/
 # Subagent-driven-development scratch (per-plan ledgers, briefs, review packages)
 .superpowers/
+# docfx output and the generated Python/C++ API pages (docfx.json, */scripts/api_markdown.py); CI builds and publishes it to gh-pages
+docs/_site/
+docs/api/
+docs/python-api/
+docs/cpp-api/

ARCHITECTURE.md

@@ -6,7 +6,7 @@ directly rather than looking for a commentary alongside it.
 
 What the code deliberately does *not* carry is the evidence behind a decision — benchmark numbers,
 memory-footprint measurements, approaches that were tried and rejected. That lives in
-[`docs/`](docs/), and [`STYLEGUIDE.md`](STYLEGUIDE.md#comments) says why.
+[`docs/`](docs/README.md), and [`STYLEGUIDE.md`](STYLEGUIDE.md#comments) says why.
 
 ## The four format families
 
@@ -47,13 +47,73 @@ source-generated map) or `ExcelParser.Build` (a runtime-configured map). Its mod
 a `ref struct` model binds directly to `Cell.Value` spans — zero allocation for the container and,
 for span-typed columns, for the values too. It consumes `Row`/`Cell` from any sheet uniformly.
 
-## Parallel CSV parsing
-
-`CsvParallel.ParseAsync<T>` (`src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsv*.cs`) is an opt-in
-alternate path over the same typed-parsing machinery, for sources large enough that partitioning
-pays for itself. `ParallelCsvFactory` decides eligibility — a seekable, UTF-8, large-enough source —
-and falls back to the ordinary sequential `ExcelParser<T>` otherwise, so callers get identical output
-either way.
+## Parallel sheet reading
+
+`SheetParallel` (`src/ExcelReader.Core/Reader/SheetParallel.cs`) reads one sheet on several threads. It is
+opt-in, takes any `IExcelSheet`, validates its arguments and dispatches on the sheet's concrete type:
+
+| sheet | engine |
+|---|---|
+| `CsvSheet` | the CSV engine (`ParallelCsvProcessor`, `ParallelCsvEnumerable`), described below. Its source comes from `CsvReader.TryGetChunkSource`, which the native layer also uses. |
+| `XlsxSheet`, `XlsbSheet` | the pipeline engine (`src/ExcelReader.Core/Parser/SheetPipeline`) |
+| anything else, and every fallback case | one sequential runner over `IExcelRowEnumerator` |
+
+The fallback cases give the same results with no error: a degree of parallelism of 1, a sheet under 256 KB
+uncompressed, XLS, a table view (`ExcelTable.AsSheet`), and a CSV that cannot be partitioned (a stream that is not a file
+or a non-UTF-8 encoding). The caller owns the workbook; the engine holds a `ReaderLifetime` lease
+until the task completes or the enumeration ends.
+
+The CSV engine is not folded into a shared partition abstraction. It is tuned, and its model, a static
+byte-range plan with speculative starts and re-reads, differs from the pipeline's dynamic, exact chunks. What
+the two share is the contract types (`IRowRecord`, `IRowAccumulator`, `RowAggregation`, `RowAction`) and the
+binders in `Parser/ParallelCsv` (`MappedProjection`, `MappedAggregation`, `MappedCallback`, `RecordAggregation`,
+`RecordCallback`), which pick the parser's CSV type map for a `CsvSheet` and the general one otherwise.
+
+### The pipeline engine
+
+```
+zip entry -> producer (1 thread) -> bounded queue -> N workers -> ordered merge
+             inflate, cut at the     row-aligned      parse +      by chunk index
+             last row boundary       chunks, indexed  row action
+```
+
+Inflate is sequential, so the engine splits the work around it instead of through it. One producer thread
+owns the entry stream, opened through `ZipIndex.OpenEntryStream` so the decompressed-byte limits count where
+they do on the sequential path. It inflates into pooled buffers and cuts each at the last row boundary,
+copying the partial tail into the next buffer; chunks default to 256 KB (`SheetPipelineSource.DefaultChunkBytes`).
+In XLSX the boundary is the last `<row` followed by a name boundary, and the producer stops at the sheet-data
+end tag, so the rest of the part is neither inflated nor buffered. XLSB has no sync marker, so the producer walks
+every record with `Biff12RecordReader` and remembers the last `BrtRowHdr`; it stops at `BrtEndSheetData`.
+
+Chunks go through a bounded `System.Threading.Channels` queue, about twice the worker count, so a full queue
+blocks the producer and peak memory is roughly capacity times chunk size whatever the sheet's size. `dop - 1`
+workers (at least one) each parse a chunk in place with the format's own enumerator over `ReadOnlyMemory<byte>`,
+call `Seed()`, then run the row action for each row. Shared strings are loaded before the pipeline starts, and a
+prologue reads up to `HeaderRow` with an ordinary enumerator, so header rows never reach a chunk. The engine's
+one primitive is an async stream of `(index, state, failure)` in chunk order, reordered through a ring of slots as
+`ParallelCsvEnumerable` does; `AggregateAsync` folds the states with `Combine`, `ForEachAsync` ignores them, and
+`ParseAsync` uses a `List<T>` per chunk and yields its items.
+
+The XLSX row boundary is a candidate, not a proof: the same bytes can sit in CDATA, a comment or malformed
+markup. So every chunk enumerator reports whether it ended cleanly, on a row boundary with no row or tag cut
+short. A run of clean chunks equals the sequential read by induction. A chunk that does not end cleanly stops
+before delivering its cut-short row; the merge keeps that chunk's rows, cancels the pipeline and reads the rest
+with one sequential enumerator, skipping the rows already delivered, and later chunks' results are dropped.
+This recovery re-inflates the sheet and only happens on such sheets. It leaves `ParseAsync` and `AggregateAsync`
+exact, but `ForEachAsync` may already have called back for dropped chunks, so it is exactly once only for a
+well-formed XLSX sheet. XLSB frames records from the first byte as the sequential reader does, so it has no
+such case.
+
+The measured gain is bounded by inflate, which stays on one thread: see "Parallel intra-sheet parsing" in
+[reader-performance.md](docs/performance/reader-performance.md). Parallel inflate would replace the producer
+only; the queue, workers and merge stay.
+
+### The CSV engine
+
+`CsvSheet` takes the older path (`src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsv*.cs`), over the same
+typed-parsing machinery, for sources large enough that partitioning pays for itself. `ParallelCsvFactory`
+decides eligibility, a seekable, UTF-8, large-enough source, and falls back to the ordinary sequential
+`ExcelParser<T>` otherwise, so callers get identical output either way.
 
 When it partitions, `CsvChunkPlan` slices the data range into four chunks per worker (not one — see
 the file's comments on why one-per-worker would buffer up to the whole output) and hands them out
@@ -96,8 +156,8 @@ Still unresolved, and untested by any of the above: whether true unordered emiss
 differently. The ordered merge is the safer default and the only mode either public overload can
 reach, so it stays absent a specific reason to revisit.
 
-`CsvParallel.ForEachAsync` exposes the same partitioning to a per-row callback rather than an
-accumulator type. It accepts a `ref struct` row, which the enumerating `CsvParallel.ParseAsync` cannot —
+`SheetParallel.ForEachAsync` exposes the same partitioning to a per-row callback rather than an
+accumulator type. It accepts a `ref struct` row, which the enumerating `SheetParallel.ParseAsync` cannot —
 not because of the interface (`IAsyncEnumerable<T>`'s `T` is declared `allows ref struct` on net9.0+),
 but because a row holding spans into a worker's buffer cannot cross to the consumer thread at all.
 

CONTRIBUTING.md

@@ -32,6 +32,14 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas
   [STYLEGUIDE.md § Untrusted Input](STYLEGUIDE.md#untrusted-input) before touching a parser: every
   length, offset, and size read from the file must be bounded before it drives an allocation.
 
+- **Guide examples compile.** A C# block in `docs/guide` marked `<!-- snippet: name -->` is copied
+  from `#region name` in `docs/snippets`, which builds with the solution. Edit the region, then
+  rewrite the guides; CI fails if they differ:
+
+  ```bash
+  python docs/snippets/sync.py
+  ```
+
 ## Commit messages
 
 Use [Conventional Commits](https://www.conventionalcommits.org/): `type(scope): summary`, imperative

Directory.Build.props

@@ -27,7 +27,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Meziantou.Analyzer" Version="3.0.290">
+    <PackageReference Include="Meziantou.Analyzer" Version="3.0.294">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>