From 2db28c82ba936763f8d8f6e0b5dc5a37305374c2 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 08:04:29 +0200 Subject: [PATCH 01/47] Add NullChannel and make string predicates null-aware (mask-based-nulls 0-1) Phase 0 of plans/mask-based-nulls.md: new ctable_nulls.py holds NullChannel, a uniform accessor for a column's validity channel, plus the shared sentinel helpers (kind_of_spec, sentinel_mask, is_nan_sentinel, is_null_value). The hand-rolled `getattr(spec, "null_value")` comparisons scattered across ctable.py, groupby.py, ctable_indexing.py, schema_validation.py and schema_vectorized.py now route through it, with no behavior change -- the existing suite passes unmodified. This incidentally unifies the "is the sentinel a NaN" test, which several sites spelled as `isinstance(nv, float)` and so missed a float32 NaN sentinel. Phase 1: string predicates over a nullable column were not null-aware at all. Only the operator form forced nulls to False; t.where("a > 10") compared the raw sentinel, so a sentinel that satisfies the predicate (null_value=999 against > 10) returned its nulls as matches. CTable._rewrite_null_predicates now conjoins a validity guard onto each comparison leaf, per leaf so that OR stays correct -- a global conjunct would drop a row that is null in one column but matches the other branch. Two refinements keep this from costing performance: the guard is emitted inline, `(a > 90) & (a != 999)`, so it stays a predicate on the same column rather than an operand opaque to the index planner; and it is emitted only when the sentinel could actually satisfy the leaf, since -1 cannot match `> 10` and NaN cannot match anything but `!=`. Under a negation the guard attaches at the negation itself, because ~((a > 10) & valid) would yield True for a null where SQL says False. Contrary to the plan, _exclude_null_positions and the indexed-OR bail are retained. An ordered index answers `a > 90` by taking a range of the sorted column and never evaluates the predicate, so a NaN sentinel -- which sorts last -- lands inside every `>` range regardless: measured on 1M rows, the scan matches 83146 rows while the FULL index returns 160070 positions, 76924 of them NaN. A null-aware expression therefore cannot make the index result correct. The plan document is corrected in place. Co-Authored-By: Claude Opus 5 --- plans/mask-based-nulls.md | 549 ++++++++++++++++++++ src/blosc2/ctable.py | 225 ++++---- src/blosc2/ctable_indexing.py | 64 ++- src/blosc2/ctable_nulls.py | 471 +++++++++++++++++ src/blosc2/groupby.py | 20 +- src/blosc2/schema_validation.py | 18 +- src/blosc2/schema_vectorized.py | 10 +- tests/ctable/test_null_channel.py | 267 ++++++++++ tests/ctable/test_null_predicate_rewrite.py | 246 +++++++++ 9 files changed, 1703 insertions(+), 167 deletions(-) create mode 100644 plans/mask-based-nulls.md create mode 100644 src/blosc2/ctable_nulls.py create mode 100644 tests/ctable/test_null_channel.py create mode 100644 tests/ctable/test_null_predicate_rewrite.py diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md new file mode 100644 index 000000000..81f8ec405 --- /dev/null +++ b/plans/mask-based-nulls.md @@ -0,0 +1,549 @@ +# Mask-based nullable columns for CTable + +> **Status: IN PROGRESS.** Phases 0 and 1 landed 2026-08-08; Phase 1's premise about the index +> path was disproven during implementation and is corrected in place (see §Expression layer). +> Drafted 2026-08-08. +> Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` +> inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled +> forward to Phase 1, sidecar suffix renamed `.notnull`. +> +> **This is the reference document for how CTable stores null values from now on.** It +> supersedes the sentinel-only decisions recorded in `plans/ctable-nulls.md` (whose non-goals +> list *"add separate validity bitmap storage for scalar CTable columns"*), +> `plans/enhancing-ctable.md` §Gap C, `plans/enhancing-ctable-phase2.md` §P5, and +> `plans/enhancing-ctable-phase3.md` §"Scope decision". Those remain accurate records of what +> was decided and shipped at the time and are not being amended; read them as history, and +> read this document for intent. + +## Context + +CTable represents nulls three different ways today: an **in-band sentinel** (`null_value`) for +numeric/bool/timestamp/string/bytes/utf8/ndarray, a **reserved code** `-1` for dictionary +columns, and **native `None`** for vlstring/vlbytes/list/struct/object. The sentinel model is +the default and it is lossy by construction — a sentinel steals a value from the dtype's range: + +- `bool(nullable=True)` silently becomes physical `uint8` with `0/1/255`, leaking raw `255` + through `col[:]` and row tuples, and requiring a whole layer of filter rewrites + (`flag == True → raw == 1`) so nulls don't leak into predicates. +- `int8`/`uint8` cannot use their full range alongside nulls. +- Free-text `utf8`/`string` has no safe sentinel — any value is legal. `blosc2.utf8(null_value="\x00")` + is *rejected outright* because NumPy 2.4 won't match a lone `"\x00"` against `StringDType`, + which would silently stop marking anything as null. +- Parquet import already carries a workaround artifact in schema metadata: + `"conversion": "nullable_scalar_wrapped_as_singleton_list"` — a nullable **bool** wrapped as a + one-element list because the sentinel couldn't represent it. +- `_compiled_columns_from_arrow` (`ctable.py:7457`) outright **raises** when a nullable Arrow + column has no available sentinel, so those types can't be imported at all. + +The goal is lossless Arrow/Parquet round-trip for every scalar type, achieved by moving nullity +into a **sidecar validity array** per column — Arrow's own model — while keeping the sentinel +path fully supported for API compatibility and giving existing tables an explicit migration. + +This design is already recorded and parked in `plans/enhancing-ctable-phase2.md` §P5 +("Mask-based nullable columns: PARKED — design recorded, do not build yet"). Its first unpark +criterion, *"a user asks for nullable bool without the 255 reservation"*, is what this work +answers. + +## Decisions (settled during planning — do not re-litigate) + +1. **Mask becomes the default** for `nullable=True` on newly created tables — in two steps: + the capability ships opt-in first (Phase 6), and the default flips no earlier than one + release later (Phase 9), so version-3-capable readers are in circulation before + default-created tables require them. Sentinel remains fully supported and readable forever, + selectable per column (`null_value=...`, `null_storage="sentinel"`) or globally via + `NullPolicy`. Existing on-disk tables keep working unchanged. +2. **V1 scope** = fixed-width scalars + utf8: numeric (incl. **complex**, which gains nullability + for the first time), bool, timestamp, `string` (`U*`), `bytes` (`S*`), `utf8`, `ndarray`. + Dictionary keeps `null_code=-1`; vlstring/vlbytes/list/struct/object keep native `None`. + `is_null()` remains the uniform user-facing API across all kinds. +3. **Layout** = plain `np.bool_` NDArray, 1 byte/row, `True = valid` (Arrow polarity), mirroring + `/_valid_rows`. Not bit-packed; `np.packbits` only at the Arrow boundary. +4. **Read semantics unchanged**: `col[:]` returns raw values with a deterministic fill in null + slots; `is_null()`/`notnull()` read the mask. Adds opt-in `col.to_numpy(masked=True)`. +5. **Fill is loud**: `float → NaN`, `timestamp → int64.min` (decodes to `NaT` for free via + `_maybe_decode_timestamp_values`), everything else dtype zero / `""` / `b""` / `0j`. + The fill is **not** part of the format contract and is **not** recorded in the schema — + recording it would recreate sentinel collisions at the metadata layer. +6. **NaN is a value, not a null**, in a mask-backed float column. Only `mask=False` is null. + This matches Arrow and is the point of a side channel. Sentinel float columns keep + NaN-as-null forever. Consequence: `dropna`/`groupby`/`min`/`max` differ between the two + storages for float — must be documented and covered by the equivalence tests. Frame the + docs positively — "mask columns follow Arrow semantics for NaN" — not as a changelog caveat. +7. **Nothing auto-migrates.** Not on `open()`, not on `copy()`, and — importantly — + not on `save()`/`to_cframe()`, which must *preserve* each column's `null_storage`. +8. **Kleene three-valued logic stays out of scope.** Masks make it possible (they supply the + validity channel `plans/enhancing-ctable.md` §Gap C named as the blocker), but + `_null_aware_compare` deliberately collapses null → `False` (SQL `WHERE` semantics). + Named deferred follow-up. +9. **An absent sidecar means all-valid.** The `.notnull` array is materialized lazily, on the + first write that actually contains a null; a mask-storage column with no `.notnull` key on + disk is a valid — and expected common — state meaning "no nulls so far". Null-free nullable + columns therefore cost zero bytes on disk and zero read-path work: `null_pred()` returns + `None`, which the expression layer already treats as never-null. It also makes + `convert_nulls(to="mask")` on a null-free sentinel column a pure schema update. + +## Architecture + +### Invariants to encode in the code, not just here + +- **Chunk pinning**: the mask array's `chunks[0]`/`blocks[0]` must equal the value column's, not + whatever `compute_chunks_blocks` picks for a bool dtype. Otherwise `_nonnull_chunks`, the + chunk-aligned writers, and index-segment alignment all re-align on every read. +- **Values under `mask=False` are unobservable** through the `Column` API. +- **Crash safety is inherited, not added**: `extend()` flips `self._valid_rows[start:end] = True` + at `ctable.py:12923`, *after* every column write. Mask writes inserted before that line are + invisible until the row goes live. Put a comment there so nobody reorders it. + +### New module: `src/blosc2/ctable_nulls.py` + +`ctable.py` is already 13.5k lines and already delegates to `ctable_storage.py` / +`ctable_indexing.py`; a fourth sibling fits. + +```text +NULL_NONE, NULL_MASK, NULL_SENTINEL, NULL_CODE, NULL_NATIVE = "none", "mask", "sentinel", "code", "native" + +class NullChannel: + """Uniform read/write accessor for one column's validity channel. + + Subsumes all four representations CTable uses -- sidecar validity array, + in-band sentinel, dictionary null code, native ``None`` -- so callers ask + *what is null* without knowing which one a column uses. + """ + __slots__ = ("_table", "_name", "_spec", "kind", "fill_value") + + def valid_array(self) # physical NDArray; mask kind only, else None + def null_pred(self) # physical LazyExpr, True where null; None if never null + def valid_pred(self) # physical, True where valid + def null_mask(self, key) # logical numpy bool; key = slice | positions | None + def valid_slice(self, a, b) # physical numpy bool for [a, b) -- Arrow export + def null_count(self) + def coerce_batch(self, values, n) # -> (values_with_fill, valid_np | None) + def coerce_scalar(self, value) # -> (storage_value, is_valid) + def set_valid(self, key, valid); def resize(self, n); def gather(self, positions) +``` + +Owned by `CTable` as `self._null_channels: dict[str, NullChannel]`, invalidated whenever +`_schema`/`col_names` change. The mask NDArrays live in `CTable._null_masks`, opened **lazily** +via `storage.open_null_mask(name)` — mirror `_LazyColumnDict` (`ctable.py:3658-3746`) rather +than opening a sidecar per column on `open()`. Creation is lazy too (decision 9): `NullChannel` +materializes the sidecar the first time `coerce_batch`/`coerce_scalar` actually reports a null; +until then `valid_array()`/`null_pred()` return `None` and the column reads as never-null. + +`Column` gets `self._nulls` and a public `Column.null_storage` property. + +This module is what keeps the work from becoming a 50-site grep, so **it lands first, +sentinel-only, with zero behavior change** (Phase 0). + +### Schema layer — `src/blosc2/schema.py`, `schema_compiler.py` + +Every spec re-declares `nullable`/`null_value` today (`_NumericSpec:94`, `timestamp:251`, +`bool:285`, `string:335`, `bytes:384`, `UTF8Spec:622`, `NDArraySpec:784`). Add a +`_NullableSpecMixin` above `_NumericSpec` with `_init_nulls(...)`, `uses_mask`/`uses_sentinel` +properties, and `_null_metadata()`; each spec's `__init__` gains one `null_storage=None` kwarg +and one call, and each `to_metadata_dict` swaps its two nullability blocks for +`**self._null_metadata()`. `_null_metadata` **never emits `"sentinel"`**, so sentinel tables +serialize byte-identically to today. + +Do *not* dodge the signature edits by post-setting `null_storage` in `spec_from_metadata_dict` — +`spec_cls(**data)` (`schema_compiler.py:447`) is the clean-fail mechanism for old readers. + +Two spec fixes fall out: +- **`bool`** (`schema.py:275-300`): drop the `null_value != 255` rejection under mask storage, and + move the `bool_ → uint8` dtype flip **out of `__init__`** into `_resolve_nullable_specs`, which + already flips there (`ctable.py:4543`). Same for `NDArraySpec` bool (`ctable.py:4510-4516`, `4545-4551`). +- **`complex64`/`complex128`** (`schema.py:208-238`): gain nullability via the mixin, fill `0j`. + +**Version gating** — `schema_to_dict` (`schema_compiler.py:487`) computes the version as an +explicit feature max: + +```python +uses_mask = any(getattr(c.spec, "null_storage", None) == "mask" for c in schema.columns) +schema_version = ( + 3 if uses_mask else (2 if schema.metadata.get("nested") is not None else 1) +) +``` + +and `schema_from_dict` accepts `(1, 2, 3)`. This deviates from P5's "no global version bump" but +preserves its actual goal — the bump is *conditional on a mask column existing*, so only +mask-using tables are unreadable by old readers. What it buys is the failure message: a readable +`ValueError: Unsupported schema version 3` instead of a `TypeError` from deep inside +`spec_cls(**data)`. Include a hint naming `convert_nulls(to="sentinel")`. + +**`NullPolicy`** (`ctable.py:117-197`) gains `null_storage: Literal["mask", "sentinel"]` +(default `"sentinel"` until Phase 9 flips it). +`CTable._resolve_nullable_specs` (`ctable.py:4496-4553`) stays the single decision point, resolving +in order: explicit `spec.null_storage` → explicit `spec.null_value` (⇒ sentinel) → +`policy.column_null_values[name]` (⇒ sentinel) → a set type-wide sentinel field matching the +column's kind (⇒ sentinel) → `policy.null_storage`. Under `"mask"` it skips sentinel selection +entirely — **no `max_length` widening, no bool dtype flip**. + +Note: once the default flips, `NullPolicy`'s type-wide sentinel fields (`string_value`, +`float_value`, `signed_int_strategy`, …) would become silently inert for plain `nullable=True`. +Do **not** have `__post_init__` raise on that combination — that would break existing working +code (`NullPolicy(float_value=...)`) on the very release that flips the default. Instead, +setting any type-wide sentinel field **implies `null_storage="sentinel"`** for the types it +covers (the resolution-order entry above); `__post_init__` raises only when +`null_storage="mask"` is passed *explicitly* alongside them, which is a genuine contradiction +the user wrote. `column_null_values` stays meaningful — it forces sentinel per column. + +Add one function `schema.fill_value_for(spec)` implementing decision 5. + +### Storage layer — `src/blosc2/ctable_storage.py` + +New key suffix beside `_UTF8_DATA_SUFFIX` (`:410`), collision-free by the same documented +argument (no column name can map to a key containing a literal `.`): + +```python +_NOTNULL_SUFFIX = ".notnull" # -> /_cols/.notnull, extension .b2nd +``` + +Named `.notnull` — deliberately **not** `.valid` — because `/_valid_rows` already exists and +means something different (row liveness vs. per-column null validity); two bool arrays with +near-identical names would invite conflation. The name also states its polarity: `True` = not +null, matching Arrow. + +Five new `TableStorage` methods (`create_null_mask` / `install_null_mask` / `open_null_mask` / +`has_null_mask` / `delete_null_mask`), implemented across all four backends exactly as +`create_valid_rows`/`open_valid_rows` (`:132-141`) already are. Per decision 9, +`has_null_mask(name) is False` is the common case and means all rows valid; `create_null_mask` +is called by `NullChannel` on the first null actually written, never at column creation: + +| backend | notes | +|---|---| +| `InMemoryTableStorage:215` | `blosc2.zeros(shape, np.bool_, ...)`; `open_*` raises as `open_column` does | +| `FileTableStorage:560` | `store[key + _NOTNULL_SUFFIX] = arr` | +| `TreeStoreTableStorage:1081` | copy the utf8-companion block at `:1304-1319` (`_dest_path`, `map_tree`, `_modified`) | +| `EmbedStoreTableStorage:413` | read-only: `open_null_mask` only; creates go in the `_not_supported` list at `:527-537` | + +The mask is a TreeStore *key* like the utf8 offsets array, so `.b2z`/`mmap_mode` are handled by +`_open_store()[key]` — no `store.offsets` special-casing (`open_varlen_scalar_column:737-741` +is the precedent). + +`delete_column` / `rename_column` already carry a bespoke `+_UTF8_DATA_SUFFIX` branch in both +`FileTableStorage` (`:901-933`) and `TreeStoreTableStorage` (`:1496-1540`). **Generalize to a loop +over `_COMPANION_KEY_SUFFIXES = (_UTF8_DATA_SUFFIX, _NOTNULL_SUFFIX)`** rather than adding a third copy. + +Capacity/persistence sites in `ctable.py`: `_grow` (`:4902`), `trim_capacity` (`:4875`), +`compact` (`:11391`), `_save_to_storage` (`:5827` — **both** the `install_column` reblock fast path +at `:5952` and the `create_column` path at `:5964`), `to_cframe` (`:5754`, per-column loop `:5809-5823`), +and **`CTable.load` (`:6099`), a second parallel open path that is easy to miss**. All of these +operate only on masks that exist — an absent sidecar needs no growing, trimming, or copying, +which is most of the lazy-materialization payoff. + +### Read / write paths + +Today fixed-width scalar columns cannot accept `None` at all — `_coerce_row_to_storage`'s +else-branch (`ctable.py:4823`) is `np.array(val, dtype=col.dtype).item()`, so users write the +sentinel literally. Under masks **`None` becomes the canonical way to write a null**, which is +new capability, not re-plumbing. `NullChannel.coerce_batch` null-detection rules: + +- Python object sequence: `v is None` → null; `pandas.NA`/`pyarrow.NA` → null (duck-typed, **not** `float('nan')`). +- NumPy float array input: NaN is a **value** (decision 6). +- `np.ma.MaskedArray` input: `~arr.mask` is the validity verbatim. +- Arrow/pandas nullable input: the source's own validity is authoritative. +- Returns `valid_np is None` when nothing was null, so callers skip the mask write entirely. + +Sites: `extend` (`:12718-12927` — coercion loop `:12845-12886`, write loop `:12898-12920`, and the +timestamp `None`-substitution at `:12855-12873`); `append` (`:12628`) plus +`_coerce_row_to_storage` returning `(dict, null_names)`; `Column.assign` (`:2503`); +`_write_arrow_batch` (`:7759`) gaining a parallel `_ChunkAlignedWriter` per masked column. + +**`Column.__getitem__` / `_values_from_key` (`:1219-1316`) do not change** — that's what makes the +read side cheap. + +**`Column.__setitem__` (`:1394-1527`) is the highest-risk function in the plan**: four key +branches × three fast paths plus two chunked loops (`:1476-1490`, `:1511-1518`). For V1, route +mask columns through the single unified `else` path (`:1491-1520`) — compute `phys_indices` +explicitly, coerce once, write values and mask together — and skip the fast paths. Measure, then +re-add them in a follow-up. Threading mask writes through all six paths at once is where this +project would break. + +### Null API — `ctable.py:2587-2721` + +`is_null()` (`:2627`) becomes `~channel.null_mask(...)`, i.e. O(1 byte/row) instead of +O(itemsize/row). `null_count()` (`:2645`) on a hole-free base table is +`n - blosc2.count_nonzero(mask[:n])` — bool NDArrays compress to near nothing, so effectively +O(chunks). `fillna` (`:2659`) becomes correct even when the fill collides with real data +(impossible under sentinels). `_nonnull_chunks` (`:2673`) zips value and mask chunks — this is +what the chunk-pinning invariant is for. New `Column.to_numpy(masked=False)` returns +`np.ma.MaskedArray` when `masked=True` and **must work for sentinel columns too** (derive the +mask from the sentinel) so the API is uniform. `_is_nullable_column` (`:13265`) becomes +`channel.kind != NULL_NONE`; `info_items` (`:1738`) gains `null_storage`. + +Struct/object columns keep the per-row Python loop at `:2638` — not made worse, worth a docs note. + +### Expression layer + +This gets *simpler*, because Phase 0 already centralized it. `Column._raw_null_pred()` +(`:1863-1881`) becomes `return self._nulls.null_pred()`, which for masks is `~mask_ndarray` +(a `LazyExpr` that composes with `&`). **`_combined_null_pred` (`:1883`), `_null_aware_arith` +(`:1899`), `_null_aware_compare` (`:1913`) and `NullableExpr` (`:859-1060`) are unchanged** — +they already consume an opaque boolean predicate. Keep null-pred (not validity) polarity so the +diff stays at zero; the single `~` fuses into the lazy expression. + +Two gains fall out: `_raw_null_pred` currently returns `None` for `is_ndarray` columns because a +per-item sentinel mask doesn't align 1:1 with rows — a mask *is* row-level, so **ndarray columns +gain null propagation in expressions for free**. And `_lazy_nonnull_mask` (`:2775-2802`) reads a +stored array instead of synthesizing a comparison, keeping the miniexpr reduction fast path with +one fewer computed operand. + +`_is_nullable_bool` (`:1824-1831`) becomes `kind == "bool" and channel.kind == NULL_SENTINEL`; +the `raw_col == 1` rewrites (`:2039`, `:2049`, `:2767`, `:13392`) go dead for mask bools and stay +alive forever for sentinel ones. + +### Arrow / Parquet + +**Export** — `iter_arrow_batches` (`:6981-7122`): hoist `valid = channel.valid_slice(start, stop)` +to the top of the per-column body, then per branch pass `mask=~valid` to `pa.array` (pyarrow packs +the numpy bool itself). The bool branch (`:7103-7107`) loses its `arr == 1`; the `U`/`S` branch +(`:7093-7102`) loses its `[None if null_mask[i] else v ...]` Python list comprehension; the ndarray +branch (`:7070-7087`) stops requiring *every element* to equal the sentinel (a lossy, surprising +rule) and uses row-level validity. Dictionary (`:7043`) and varlen/list/struct (`:7038`) are unchanged. + +`UTF8Array.arrow_slice` (`_utf8_array.py:961-984`) is the **one** place `np.packbits` is genuinely +needed, because it builds Arrow buffers directly: + +```python +validity = pa.py_buffer(np.packbits(valid, bitorder="little")) # bitorder is MANDATORY +``` + +Arrow validity bitmaps are LSB-first. This is the easiest bug in the plan to introduce and the +hardest to catch — a round-trip test passes with **either** bit order if import unpacks the same +way. Pin it with a test asserting the literal packed bytes for a known pattern. + +**Import** — in `_compiled_columns_from_arrow` (`:7369-7482`), when the resolved storage is +`"mask"` the entire sentinel-selection block is skipped and **the `"no null_value sentinel is +available"` error at `:7457` never fires**. That single deletion is what makes nullable bool, +full-range `int8`/`uint8`, and free-text utf8 importable at all. Add a `null_storage=` parameter +to `from_arrow`/`from_parquet` beside the nullable knob at `:8349`, defaulting to the policy. + +New `_arrow_column_to_numpy_masked(arrow_col, col) -> (values, valid)` beside +`_arrow_column_to_numpy` (`:7794-7828`): `arrow_col.fill_null(fill_value_for(spec)).to_numpy(zero_copy_only=False)` +(no Python loop, and the fill is exactly ours), with `valid = arrow_col.is_valid().to_numpy(...)` +for correctness first — optimize to `np.unpackbits(..., bitorder="little", count=n)` off the raw +validity buffer later, honoring `arrow_col.offset` and chunking. + +When extending an **existing** table from Arrow, the stored schema's `null_storage` wins; only +inferred schemas consult the policy. + +**Round-trip contract** (stated and tested): `to_arrow(from_arrow(x)).equals(x)` exactly, for +nullable `bool`; `int8`/`uint8` using **all 256 values** plus nulls; `float64` containing `nan`, +`±inf`, `0.0`, `-0.0` **as values** plus separate nulls; `utf8` containing `""`, `"\x00"`, +`"__BLOSC2_NULL__"`, 4-byte UTF-8 plus nulls; `timestamp` with `int64.min` as a value plus +separate nulls; `string(max_length=4)`/`bytes(max_length=4)` fully occupying the width. **None of +these round-trip under sentinels.** Same list for Parquet. + +### Reductions and summary indexes + +Mechanical: `_ndarray_values_for_reduction` (`:2843-2867`) and `argmin`/`argmax` (`:3218-3271`) +swap `_null_mask_for` for `channel.null_mask`. + +**`_summary_minmax_source` (`:3004-3068`) is NOT fixed by masks** — verified. The bail at `:3046` +(`if nullable and not is_nan_float`) exists because a non-NaN sentinel pollutes per-block extrema, +and the **fill value pollutes them identically**: the summary builder reads the physical array and +never consults a side channel. Two honest routes: + +1. *Free, partial*: with the NaN float fill (decision 5), mask-backed float columns qualify under + the existing `is_nan_float` escape hatch at `:3045` with a one-condition change. Floats only — + `int64.min` **is** the block minimum, so timestamps get nothing free. +2. *Real fix (Phase 10)*: make the summary builder in `ctable_indexing.py` mask-aware — extrema over + `values[valid]`, a per-segment `all_null` flag, `"null_aware": true` in the descriptor. This + retroactively enables the fast path for **sentinel** columns too. + +Until (2) lands, mask columns take the same bail as sentinel columns. **Do not ship a fast path +that is silently wrong.** + +### Sort and indexes + +`_build_lex_keys` (`:11653-11730`): the null-indicator key becomes `(~valid[live_pos]).astype(np.intp)` — +cheaper than a sentinel compare, and no string comparison for `U`/`S`. Semantics unchanged. + +`_sorted_positions_from_full_index` (`:11519-11651`): line `:11633` currently reads the **entire raw +column** just to compute `null_phys`; with a mask that becomes `~np.asarray(mask[:])`, an 8×–64× +I/O reduction on exactly the path whose comment at `:11631` apologizes for its temporaries. +**Highest value-per-line change in the plan.** + +`_utf8_rank_arrays` (`ctable_indexing.py:99-143`) — verified landmine. Line `:120` is +`is_null = uniques == null_value`; under masks there's no sentinel in the vocabulary and the `""` +fill factorizes as an ordinary entry with **rank 0, so nulls would sort first and collide with +genuine empty strings**. New signature `_utf8_rank_arrays(col, n_phys, null_value=None, *, valid=None)` +with `ranks[~valid] = null_rank` after the `code_to_rank[codes]` gather, and `"null_aware": True` +in the returned meta. Comment the hazard in place. `_DictRankWrapper` (`:167-209`) is unchanged. + +Index descriptors gain `{"null_aware": true, "null_order": "last"}` (anticipated by +`plans/ctable-nulls.md:614-623`, present in neither `plans/ctable-indexes-opsi.md` nor the code). +Read with `.get("null_aware", False)`; bump the build token so stale indexes rebuild. + +**The indexed-OR bail (`ctable_indexing.py:1459-1461`) is not fixed by masks either** — the problem +is that global post-filtering (`_exclude_null_positions:1498-1509`) drops rows that legitimately +match via the *other* branch. The right fix is per-leaf and **independent of storage**: add +`_rewrite_null_predicates(expr, operands)` alongside `_rewrite_dictionary_predicates` (`:12953`) +and `_rewrite_utf8_predicates` (`:13158`), rewriting each nullable comparison leaf `a > 10` into +`(a > 10) & _valid_a` (`~mask` for mask columns, `a != nv` for sentinel ones). Because this helps +sentinel columns that exist today and is independent of storage, **it is pulled forward: it is +Phase 1**, landing right after the `NullChannel` refactor and before any mask work. + +> **Correction (implemented 2026-08-08).** This paragraph originally claimed that "AND and OR both +> become correct with no post-filter", so `_exclude_null_positions` and the +> `nullable_indexed`/`nullable_needs_exclude` bookkeeping at `:1443-1457` would disappear. +> **That is wrong, and Phase 1 shipped with the post-filter retained.** An ordered index does not +> *evaluate* the predicate: it answers `a > 90` by taking a **range of the sorted column**, and the +> sentinel lies inside that range whether or not it would satisfy the comparison. Measured on 1M +> rows with a NaN sentinel: the scan matches 83 146 rows, while the FULL index returns 160 070 +> positions, 76 924 of them NaN. Making the *expression* null-aware therefore cannot make the +> *index result* correct — the post-filter is load-bearing, and the indexed-OR bail stays too. +> Genuinely indexed OR over a nullable column needs the index itself to know about nulls, which is +> Phase 10 (`null_aware`/`null_order` descriptors), not something the expression layer can deliver. +> +> What Phase 1 did land is larger than this paragraph anticipated: **string predicates were not +> null-aware at all**. Only the operator form (`Column._null_aware_compare`) forced nulls to False; +> `t.where("a > 10")` compared the raw sentinel, so any sentinel that satisfies the predicate +> (`null_value=999` against `> 10`) returned its nulls as matches — on the scan path, indexed or +> not. That is the bug the rewrite fixes, and it makes the scan fallback correct, which is what +> finally makes the *bailing* OR path return the right answer. +> +> Two refinements the original text did not have: +> +> - **Guards are emitted inline, not as an injected operand.** `(a > 90) & (a != 999)` keeps the +> guard a predicate on the same column; an opaque extra boolean operand pushed the planner off +> the index onto a full scan (measured ~13 % slower on the OR probe). +> - **A guard is emitted only when the sentinel could actually satisfy the leaf.** `-1` cannot +> match `> 10`, and NaN cannot match anything but `!=`, so those leaves are left alone. Without +> this, every nullable-column query loses its index. Never applied inside a negation: a sentinel +> that *fails* `a > 10` *passes* `not (a > 10)`. +> +> **Negation caveat, resolved.** The original text proposed either bailing on `~`/`not` or pushing +> validity to the outermost conjunction. Bailing was rejected: the index path is SQL-correct for +> negation today while the scan path is not, so bailing would have regressed the index. Validity is +> pushed to the negation point — `(~(a > 10)) & valid_a` — which is exact for every tested form and +> conservative in one three-valued corner: `not (a > 10 and b == 999)` with a false second term +> makes SQL's `NULL AND FALSE` collapse to `FALSE`, so the row should survive, while the guard drops +> it. Rows are only ever dropped, never wrongly returned. + +### Groupby + +`groupby.py:_null_mask` (`:2116-2137`) is already the single central helper, but it receives an +already-gathered `values` chunk, so validity has to be gathered at each of ~8 independent read +sites: `:534`, `:583-589`, `:624-637`, `:716-718`, `:1005-1017`, `:1394`, `:1430`, `:1757`, plus +`_null_value_for` (`:2589`) and `_null_output_value` (`:2239`), which for a mask-backed output +column become "write fill + mask=False". Signature grows `*, valid=None`. **Messiest integration +after `__setitem__`** — budget accordingly. + +One semantic improvement follows from decision 6: NaN in a float *value* column is no longer +missing. Keep the `is_key` NaN coercion at `:2131-2134` for keys (dropna semantics). + +### Nullable-bool cleanup + +Under masks, `bool(nullable=True)` yields physical `np.bool_` — no `uint8`, no reserved `255`. +`t.flag[:]` returns booleans; `t.where(t.flag)` works directly; export is +`pa.array(arr, mask=~valid, type=pa.bool_())`. + +**No deprecation cycle is needed, and that's the point of the design**: the change is scoped by +construction to tables created *after* the default flips. Existing tables carry `null_value: 255` +in their schema and stay uint8 forever; `blosc2.bool(nullable=True, null_value=255)` and +`null_storage="sentinel"` keep producing uint8 explicitly; `_is_nullable_bool` and its rewrite +sites stay permanently. What is needed: the two dtype-flip relocations, a +`doc/reference/ctable.rst` note, a release-note entry, and a test that opens a checked-in fixture +table with a uint8 nullable-bool column and asserts nothing changed. + +### Migration + +```python +def convert_nulls( + self, columns=None, *, to="mask", null_value=None, inplace=False +) -> CTable: + """Convert nullable columns between sentinel and validity-mask storage. + + Never called implicitly: opening, copying, and saving a table all preserve + each column's existing null storage. + """ +``` + +Per kind for `to="mask"`: a null-free sentinel column (no slot holds the sentinel) converts as +a **pure schema update** — no sidecar is written, per decision 9. Otherwise +numeric/timestamp/string/bytes convert chunkwise in one pass +(`valid = ~sentinel_mask(chunk)`; overwrite sentinel slots with the fill), dtype unchanged, +in-place possible — but `string`/`bytes` keep their widened `max_length` (shrinking is a dtype +change; document `copy()` as the way to reclaim it, don't shrink silently). `bool` needs +uint8 → `np.bool_`, i.e. a new array. `ndarray` derives the row mask under the old all-elements +rule first. `utf8` rewrites null rows to zero-length spans (effectively a column rebuild). +Dictionary and varlen/list/struct/object are **no-ops** — raise if named explicitly, skip silently +if implicit. + +`to="sentinel"` is the inverse and must **reject** what it cannot represent: full-range +`int8`/`uint8`, and utf8/string whose data already contains the proposed sentinel. Check before +writing; raise naming the offending value. + +`inplace=True` ordering **is** the crash-safety argument, so put it in the docstring: (1) write +the complete `.notnull` sidecar, (2) rewrite value slots to fill, (3) update `/_meta` schema last. +A crash after (1) or (2) leaves the schema saying `sentinel`, the orphan `.notnull` key unread, and +the table intact. `inplace=False` (default, recommended) builds a new table via `copy()`. + +Detection is `Column.null_storage` plus an `info()` column — no separate report function. + +## Verification + +**Differential oracle — the single highest-value test.** New +`tests/ctable/test_null_storage_equivalence.py`: build the same logical data twice +(sentinel-backed and mask-backed), assert every public API agrees — `is_null`, `notnull`, +`null_count`, `fillna`, `dropna`, `sort_by` (both directions, single and multi-key), `group_by` +(as key and as value, `dropna=True/False`), `where` (indexed and unindexed, AND and OR), all +reductions, `argmin`/`argmax`, `unique`, `value_counts`, `to_arrow`, `to_pandas` — parametrized +over every V1 kind, with float NaN cases marked as intentionally divergent per decision 6. This +converts "mask is a second implementation" from a permanent liability into a checked invariant. + +Additions to existing suites: +- `tests/ctable/test_nullable.py` (756 → ~1150): parametrize over `null_storage`; mask-only cases + for nullable bool `dtype == np.bool_`, full-range `int8`, utf8 containing `"__BLOSC2_NULL__"`/`""`/`"\x00"`, + `string(max_length=4, nullable=True)` staying `U4` (regression against the widening at `:4538`), + fill determinism, `to_numpy(masked=True)` for both storages, `null_count` with deletions, + `is_null()` on sorted and `where()` views, `fillna` with a value equal to real data, and `None` + accepted by `append`/`extend`/`__setitem__`/`assign` for every V1 kind. Plus the decision-9 + invariant: a mask column written with no nulls has **no** `.notnull` key on disk and answers + `is_null()` all-`False`; the key appears exactly when the first null is written. And the + `NullPolicy` resolution rules: type-wide sentinel fields imply sentinel storage for the types + they cover; explicit `null_storage="mask"` combined with them raises. +- `test_null_expressions.py` (284 → ~440): mask/sentinel differential for arithmetic and comparison + propagation; ndarray-column propagation (new capability); NaN-is-a-value assertions. +- `test_arrow_interop.py` (630 → ~840) and `test_parquet_interop.py` (1323 → ~1500): the round-trip + contract list exhaustively, plus the literal-bytes bit-order test. + +New files: +- `tests/ctable/test_null_persistence.py`: mask survives `.b2d`, `.b2z`, `to_cframe`/`ctable_from_cframe`, + inline TreeStore save/load, `mmap_mode="r"`, `CTable.load()`, `delete_column`, `rename_column`, + `compact()`, `trim_capacity()`, and repeated `_grow()` cycles. +- `tests/ctable/test_null_migration.py`: `convert_nulls` per kind both directions, in-place and copy; + the null-free schema-only fast path; crash-ordering (an orphan `.notnull` key with an unmodified + schema still reads as sentinel); and the two guarantees — old tables open unchanged, `save()` + preserves storage under a mask-default policy. +- A version-gate test: hand-build a `version: 3` schema dict and assert a simulated old accept-list + `(1, 2)` raises a clear `ValueError` naming the version. + +End-to-end smoke, run manually: import a nullable-bool + full-range-`int8` + free-text-utf8 Parquet +file, round-trip it, and assert `pq.read_table(out).equals(pq.read_table(in))` — the case that is +impossible today. Also re-run the OFF importer round-trip (`plans/ctable-nulls.md` §Tests) and +confirm the `nullable_scalar_wrapped_as_singleton_list` workaround can be deleted. + +## Phasing + +Each phase is independently landable. **The default does not flip until Phase 9**, which is a +deliberate one-release-minimum lag behind Phase 6 so version-3-capable readers circulate before +default-created tables require them. + +| # | Phase | Size | Risk | +|---|---|---|---| +| 0 | ✅ **`NullChannel` refactor, sentinel-only.** New `ctable_nulls.py`; route the ~40 `getattr(spec, "null_value")` sites through it across `ctable.py`, `groupby.py`, `ctable_indexing.py`, `schema_validation.py`, `schema_vectorized.py`. Test suite must pass **unmodified**. | M | Low | +| 1 | ✅ **Per-leaf null-predicate rewrite** *(storage-independent — pulled forward because it fixes sentinel tables that exist today)*. `_rewrite_null_predicates`, guards emitted inline and only where the sentinel could satisfy the leaf; validity pushed to the negation point. **`_exclude_null_positions` and the indexed-OR bail are retained** — see the correction above; an ordered index never evaluates the predicate, so a null-aware expression cannot fix it. Real payoff: string predicates become null-aware at all. | M | Med | +| 2 | **Schema plumbing.** `_NullableSpecMixin`, `null_storage` kwarg on ~9 specs, conditional version 3, `NullPolicy.null_storage` (**still defaulting to `"sentinel"`**) with sentinel-field inference, `_resolve_nullable_specs` branch, bool/ndarray dtype-flip relocation, `fill_value_for`, complex nullable. | S | Low | +| 3 | **Storage sidecar.** 5 methods × 4 backends, lazy creation (absent key = all valid); `_grow`/`trim_capacity`/`compact`/`_save_to_storage`/`to_cframe`/`load`; companion-suffix loop in delete/rename. Testable with a hand-built mask, no semantics yet. | M | Low | +| 4 | **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`. Mask columns fully usable. | **L** | **High** (`__setitem__`) | +| 5 | **Expressions + reductions.** `_raw_null_pred`, `_lazy_nonnull_mask`, `_ndarray_values_for_reduction`, argmin/argmax, `_is_nullable_bool`. Includes the ndarray-propagation gain. | M | Med | +| 6 | **Arrow/Parquet.** Import + export for all V1 kinds, `packbits`/`unpackbits` LSB-first, `arrow_slice(validity=)`, delete the "no sentinel available" import error. Ships **opt-in** (`null_storage="mask"`); the default stays `"sentinel"`. | M | Med | +| 7 | **Sort + groupby.** `_build_lex_keys`, `_sorted_positions_from_full_index` (big I/O win), `_utf8_rank_arrays(valid=)`, groupby `_null_mask` threading. | M | Med-High | +| 8 | **Migration + docs.** `convert_nulls`, `Column.null_storage`, `info()`, `doc/reference/ctable.rst` null-policy rewrite, release notes. | S–M | Low | +| 9 | **Default flips to `"mask"`.** A one-line `NullPolicy` change plus release notes — lossless round-trip is why the default exists. Lands **no earlier than one release after Phase 6** so older readers in the wild already understand schema version 3. | S | Low | +| 10 | **Index null-awareness remainder** *(independent)*. Mask-aware summary builder; `null_aware`/`null_order` descriptors; re-enable `_summary_minmax_source` for mask and sentinel columns alike. | **L** | High | + +The riskiest, most-coupled work is isolated into Phases 4, 7 and 10, each of which can slip +without blocking the others. Phase 9 is a policy change, not code — its only prerequisite is +that Phases 2–8 have soaked for a release. diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 9f0eef5b8..f2159f8e4 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -34,6 +34,15 @@ import blosc2 from blosc2 import compute_chunks_blocks from blosc2.ctable_indexing import _CTableIndexingMixin +from blosc2.ctable_nulls import ( + NULL_SENTINEL, + NullChannel, + is_nan_sentinel, + kind_of_spec, + rewrite_null_predicates, + sentinel_guard_expr, + sentinel_mask, +) from blosc2.ctable_storage import ( FileTableStorage, InMemoryTableStorage, @@ -1067,6 +1076,19 @@ def __init__(self, table: CTable, col_name: str, mask=None): self._table = table self._col_name = col_name self._mask = mask + self._null_channel = None + + @property + def _nulls(self) -> NullChannel: + """This column's validity channel (see :mod:`blosc2.ctable_nulls`). + + Built on first use and kept for this ``Column``'s lifetime. The + channel reads through to the live schema on every access, so it stays + correct even if the column's spec is mutated in place. + """ + if self._null_channel is None: + self._null_channel = NullChannel(self) + return self._null_channel @property def _raw_col(self): @@ -1823,11 +1845,17 @@ def _unwrap_operand(other): @property def _is_nullable_bool(self) -> bool: - col = self._table._schema.columns_by_name.get(self._col_name) + """True for a bool column whose nulls are the in-band ``255`` sentinel. + + Such a column is physically ``uint8``, so predicates over it must be + rewritten (``flag == True`` -> ``raw == 1``) to keep the sentinel from + reading as truthy. + """ + spec = self._nulls.spec return ( - col is not None - and col.spec.to_metadata_dict().get("kind") == "bool" - and getattr(col.spec, "null_value", None) is not None + spec is not None + and spec.to_metadata_dict().get("kind") == "bool" + and self._nulls.kind == NULL_SENTINEL ) @property @@ -1862,23 +1890,12 @@ def _coerce_timestamp_operand(self, other): def _raw_null_pred(self): """Boolean lazy predicate over the raw physical array, True where the - value is this column's null sentinel. + value is null. - Returns ``None`` when there is nothing to propagate: no ``null_value`` - configured, or a fixed-shape ndarray column (whose per-item sentinel - mask does not align 1:1 with the row-level predicates built here; - see ``Column.is_null()`` for those instead). Dictionary and - variable-length scalar columns never reach here because - ``_ensure_queryable`` already rejects them for arithmetic/comparisons. + Returns ``None`` when there is nothing to propagate; see + :meth:`NullChannel.null_pred` for exactly when. """ - if self.is_ndarray: - return None - nv = self.null_value - if nv is None: - return None - if isinstance(nv, (float, np.floating)) and np.isnan(nv): - return blosc2.isnan(self._raw_col) - return self._raw_col == nv + return self._nulls.null_pred() def _combined_null_pred(self, other): """OR of self's and other's raw null predicates; ``None`` if neither @@ -2591,10 +2608,16 @@ def _assign_varlen_scalar(self, data) -> None: @property def null_value(self): """The sentinel value that represents NULL for this column, or ``None``.""" - col_info = self._table._schema.columns_by_name.get(self._col_name) - if col_info is None: - return None - return getattr(col_info.spec, "null_value", None) + return self._nulls.sentinel + + @property + def null_storage(self) -> str: + """How this column represents its nulls. + + One of ``"none"``, ``"sentinel"``, ``"code"`` (dictionary) or + ``"native"`` (variable-length containers holding ``None`` cells). + """ + return self._nulls.kind def _null_mask_for(self, arr: np.ndarray) -> np.ndarray: """Return a bool array True where *arr* contains the null sentinel. @@ -2602,62 +2625,31 @@ def _null_mask_for(self, arr: np.ndarray) -> np.ndarray: Always returns an array of the same length as *arr*; all False when no null_value is configured. """ - nv = self.null_value - if nv is None: - return np.zeros(len(arr), dtype=np.bool_) - arr = np.asarray(arr) - if self.is_ndarray: - if arr.ndim <= self.item_ndim: - arr = arr.reshape((1, *arr.shape)) - if isinstance(nv, (float, np.floating)) and np.isnan(nv): - elem_mask = np.isnan(arr) - else: - elem_mask = arr == nv - inner_axes = tuple(range(1, elem_mask.ndim)) - return elem_mask.all(axis=inner_axes) if inner_axes else elem_mask.astype(np.bool_) - if np.issubdtype(arr.dtype, np.datetime64): - # Timestamp columns materialize with the int64 sentinel already - # decoded into np.datetime64('NaT') (they share the same bit - # pattern), so the sentinel value itself never appears in arr. - return np.isnat(arr) - if isinstance(nv, (float, np.floating)) and np.isnan(nv): - return np.isnan(arr) - return arr == nv + return self._nulls.mask_for_values(arr) def is_null(self) -> np.ndarray: - """Return a boolean array True where the live value is the null sentinel. + """Return a boolean array True where the live value is null. For varlen scalar columns (vlstring/vlbytes) nullability is represented as native ``None`` values, so this returns True wherever the value is ``None``. For dictionary columns, returns True where the code equals the null_code (``-1`` by default). """ - if self.is_dictionary: - return self._dictionary_eq(None) - if self.is_varlen_scalar and not self.is_utf8: - return np.array([v is None for v in self], dtype=np.bool_) - return self._null_mask_for(self[:]) + return self._nulls.null_mask() def notnull(self) -> np.ndarray: - """Return a boolean array True where the live value is *not* the null sentinel.""" + """Return a boolean array True where the live value is *not* null.""" return ~self.is_null() def null_count(self) -> int: - """Return the number of live rows whose value equals the null sentinel. + """Return the number of live rows whose value is null. - Returns ``0`` in O(1) if no ``null_value`` is configured for this column - and the column is not a varlen scalar column. + Returns ``0`` in O(1) if this column has no null channel at all. """ - if self.is_dictionary: - return int(self.is_null().sum()) - if self.is_varlen_scalar and not self.is_utf8: - return sum(1 for v in self if v is None) - if self.null_value is None: - return 0 - return int(self.is_null().sum()) + return self._nulls.null_count() def fillna(self, value): - """Return live values with null sentinels replaced by *value*. + """Return live values with nulls replaced by *value*. Dictionary and variable-length scalar columns (whose nulls are native ``None`` cells) return a list; other columns return a NumPy @@ -2666,30 +2658,17 @@ def fillna(self, value): if (self.is_dictionary or self.is_varlen_scalar) and not self.is_utf8: return [value if v is None else v for v in self[:]] arr = np.array(self[:], copy=True) - if self.null_value is not None: - arr[self._null_mask_for(arr)] = value + if self._nulls.kind == NULL_SENTINEL: + arr[self._nulls.mask_for_values(arr)] = value return arr def _nonnull_chunks(self): """Yield chunks of live, non-null values. - Each yielded array has the null sentinel values removed. If no - null_value is configured this behaves identically to - :meth:`iter_chunks`. + Each yielded array has the null values removed. If this column has no + null channel this behaves identically to :meth:`iter_chunks`. """ - nv = self.null_value - if nv is None: - yield from self.iter_chunks() - return - is_nan_nv = isinstance(nv, float) and np.isnan(nv) - for chunk in self.iter_chunks(): - if is_nan_nv: - mask = ~np.isnan(chunk) - else: - mask = chunk != nv - filtered = chunk[mask] - if len(filtered) > 0: - yield filtered + yield from self._nulls.nonnull_chunks() def unique(self) -> np.ndarray: """Return sorted array of unique live, non-null values. @@ -2791,12 +2770,8 @@ def _lazy_nonnull_mask(self, where=None): mask = None if all_rows_visible else self._lazy_valid_rows() if where is not None: mask = where if mask is None else mask & where - nv = self.null_value - if nv is not None: - if isinstance(nv, (float, np.floating)) and np.isnan(nv): - nonnull = ~blosc2.isnan(raw) - else: - nonnull = raw != nv + nonnull = self._nulls.valid_pred() + if nonnull is not None: mask = nonnull if mask is None else mask & nonnull return mask @@ -3042,7 +3017,7 @@ def _summary_minmax_source(self): spec = col.spec if col is not None else None nullable = getattr(spec, "nullable", False) null_value = getattr(spec, "null_value", None) - is_nan_float = dtype.kind == "f" and isinstance(null_value, float) and np.isnan(null_value) + is_nan_float = dtype.kind == "f" and is_nan_sentinel(null_value) if nullable and not is_nan_float: return None # non-NaN sentinel leaks into the block extrema root = table._root_table @@ -3441,11 +3416,10 @@ def __init__(self, table: CTable, prefix: str, leaves: list[str]): def _leaf_is_null_at_logical(self, leaf: str, idx: int) -> bool: col = self._table[leaf] v = col[idx] - nv = col.null_value - if nv is None: + if col._nulls.kind != NULL_SENTINEL: return v is None try: - return bool(col._null_mask_for(np.asarray([v]))[0]) + return bool(col._nulls.mask_for_values(np.asarray([v]))[0]) except Exception: return v is None @@ -11631,10 +11605,7 @@ def _sorted_positions_from_full_index(self, name: str, ascending: bool) -> np.nd # Free each 24M-element temporary as soon as it is consumed to keep # peak memory near the size of the permutation itself. raw = np.asarray(root._cols[name][:]) - if isinstance(null_value, float) and np.isnan(null_value): - null_phys = np.isnan(raw) - else: - null_phys = raw == null_value + null_phys = sentinel_mask(raw, null_value) del raw if null_phys.any(): is_null = null_phys[positions] @@ -11721,11 +11692,7 @@ def _build_lex_keys( if is_dict_col and col_info.spec.nullable: lex_keys.append(is_null.astype(np.intp)) elif nv is not None: - if isinstance(nv, float) and np.isnan(nv): - null_ind = np.isnan(raw).astype(np.intp) - else: - null_ind = (raw == nv).astype(np.intp) - lex_keys.append(null_ind) + lex_keys.append(sentinel_mask(raw, nv).astype(np.intp)) return lex_keys @@ -11947,7 +11914,7 @@ def _null_block_bounds(self, full: dict, null_value, n: int) -> tuple[int, int]: from blosc2.indexing import _open_sidecar_file vnd = _open_sidecar_file(full["values_path"]) - if isinstance(null_value, float) and np.isnan(null_value): + if is_nan_sentinel(null_value): # NaN sorts last and breaks ordered comparisons, so count the trailing # block directly, one chunk at a time (peak memory = a single chunk). chunk = int(vnd.chunks[0]) if vnd.chunks else len(vnd) @@ -13028,6 +12995,60 @@ def in_repl(match: re.Match, _dc=dc, _name=name) -> str: rewritten = new_expr return rewritten, new_operands + def _rewrite_null_predicates( + self, expr: str, operands: dict[str, blosc2.NDArray | blosc2.LazyExpr] + ) -> tuple[str, dict[str, blosc2.NDArray | blosc2.LazyExpr]]: + """Make a string predicate reject the nulls of the columns it reads. + + The operator form (``t.where(t.a > 10)``) has always been null-aware: + :meth:`Column._null_aware_compare` forces null rows to False. The + string form was not -- ``t.where("a > 10")`` compared the raw sentinel, + so a column whose sentinel happens to satisfy the predicate (say + ``null_value=999`` against ``> 10``) returned its nulls as matches. + + Each nullable column referenced by the expression contributes a + validity operand (``a != null_value``, or ``~isnan(a)`` for a NaN + sentinel), which :func:`~blosc2.ctable_nulls.rewrite_null_predicates` + conjoins onto every comparison that reads it. The validity operand is + a lazy expression over the same raw array, so it fuses into the same + pass rather than materializing anything. + + Making the expression itself null-aware is what lets the index path + drop its own null post-filtering: index and scan now answer from the + same predicate instead of the index correcting for the scan. + + Run this *before* :meth:`_rewrite_nested_expression`, while names in + *expr* are still real column names; the injected operand names carry no + dot, so the nested rewrite passes over them. + """ + valid_exprs: dict[str, tuple[str, object]] = {} + new_operands = dict(operands) + for i, name in enumerate(operands): + if name in self._computed_cols: + continue + col_info = self._schema.columns_by_name.get(name) + if col_info is None or kind_of_spec(col_info.spec) != NULL_SENTINEL: + continue + if not self._expression_references_name(expr, name): + continue + guard = sentinel_guard_expr(name, col_info.spec.null_value) + if guard is None: + # A sentinel with no literal form: fall back to an injected + # boolean operand. Correct, but opaque to the index planner. + valid_pred = self[name]._nulls.valid_pred() + if valid_pred is None: + continue + guard = f"__nv{i}" + new_operands[guard] = valid_pred + valid_exprs[name] = (guard, col_info.spec.null_value) + + rewritten = rewrite_null_predicates(expr, valid_exprs) + if rewritten is None: + return expr, operands + return rewritten, { + k: v for k, v in new_operands.items() if not k.startswith("__nv") or k in rewritten + } + @staticmethod def _alias_dotted(expr: str, names: list[str], prefix: str) -> tuple[str, dict[str, str]]: """Replace each dotted name in *expr* with ``{prefix}{i}``. @@ -13263,8 +13284,7 @@ def _utf8_span_eval( ) def _is_nullable_column(self, name: str) -> bool: - col = self[name] - return col.null_value is not None or col.is_dictionary or col.is_varlen_scalar + return self[name]._nulls.is_nullable def dropna(self, subset: list[str] | None = None) -> CTable: """Return a view excluding rows where any column in *subset* is null. @@ -13383,6 +13403,7 @@ def where( # noqa: C901 utf8_names = self._utf8_names_in(expr_result) operands = self._where_expression_operands(expr_result) expr_result, operands = self._rewrite_dictionary_predicates(expr_result, operands) + expr_result, operands = self._rewrite_null_predicates(expr_result, operands) expr_result, operands = self._rewrite_nested_expression(expr_result, operands) expr_result = self._lazyexpr_over_cols(expr_result, operands, utf8_names) if isinstance(expr_result, np.ndarray) and expr_result.dtype == np.bool_: diff --git a/src/blosc2/ctable_indexing.py b/src/blosc2/ctable_indexing.py index 800f17dc2..38a7f47ae 100644 --- a/src/blosc2/ctable_indexing.py +++ b/src/blosc2/ctable_indexing.py @@ -20,6 +20,7 @@ import blosc2 from blosc2 import compute_chunks_blocks +from blosc2.ctable_nulls import NULL_SENTINEL, is_nan_sentinel, kind_of_spec, sentinel_mask from blosc2.schema import ( DictionarySpec, ListSpec, @@ -1440,23 +1441,35 @@ def _try_index_where(self, expr_result: blosc2.LazyExpr) -> np.ndarray | None: return None primary_col_name, primary_col_arr, _ = indexed_columns[0] + # Null exclusion still happens here, even though every predicate that + # reaches this point is null-aware (CTable._rewrite_null_predicates for + # the string form, Column._null_aware_compare for the operator one). + # An ordered index does not *evaluate* the predicate: it answers + # ``a > 90`` by taking a range of the sorted column, and the sentinel + # lives in that range whether or not it would satisfy the comparison -- + # a NaN sentinel sorts last, so every NaN row comes back for any + # ``>`` query. The expression being correct therefore does not make the + # index result correct, and these positions must still be filtered. nullable_indexed = [ name for name, _arr, _descriptor in indexed_columns - if getattr(root._schema.columns_by_name[name].spec, "null_value", None) is not None + if kind_of_spec(root._schema.columns_by_name[name].spec) == NULL_SENTINEL ] - # A NaN null sentinel can never satisfy a comparison (every comparison - # with NaN is False), so the predicate itself already drops those rows; - # only non-NaN sentinels (which *can* match a predicate) need explicit - # position-based exclusion. This lets the fast mask-direct path apply to - # NaN-nullable columns instead of falling back to the positions path. - nullable_needs_exclude = [] - for name in nullable_indexed: - nv = getattr(root._schema.columns_by_name[name].spec, "null_value", None) - if not (isinstance(nv, float) and np.isnan(nv)): - nullable_needs_exclude.append(name) - - # Global null post-filtering is not correct for OR expressions. + # Only non-NaN sentinels need the *positions* fall-back below; for a NaN + # sentinel the mask-direct path can stay, because that path evaluates the + # predicate through miniexpr rather than reading an ordered range. + nullable_needs_exclude = [ + name + for name in nullable_indexed + if not is_nan_sentinel(root._schema.columns_by_name[name].spec.null_value) + ] + + # Global null post-filtering is not correct for OR expressions: it would + # drop a row that is null in one column but matches the other branch. + # (The per-leaf rewrite makes the *expression* handle OR correctly; it + # cannot fix an index that never evaluates the expression, so an OR over + # a nullable indexed column still falls back to the scan -- which is now + # itself null-aware, and so now returns the right answer.) if nullable_indexed and ("|" in expr_result.expression or " or " in expr_result.expression): return None @@ -1498,14 +1511,9 @@ def _try_index_where(self, expr_result: blosc2.LazyExpr) -> np.ndarray | None: def _exclude_null_positions(positions): positions = np.asarray(positions, dtype=np.int64) for name in nullable_indexed: - col = root._schema.columns_by_name[name] + nv = root._schema.columns_by_name[name].spec.null_value raw = root._cols[name][positions] - nv = getattr(col.spec, "null_value", None) - if isinstance(nv, float) and np.isnan(nv): - keep = ~np.isnan(raw) - else: - keep = raw != nv - positions = positions[keep] + positions = positions[~sentinel_mask(raw, nv)] return positions if plan.exact_positions is not None: @@ -1542,23 +1550,13 @@ def _exclude_null_positions(positions): raw = np.asarray(raw) if hasattr(raw, "__array__") else raw pos = candidates for name in nullable_indexed: + nv = root._schema.columns_by_name[name].spec.null_value if name == primary_col_name: - nv = getattr(root._schema.columns_by_name[name].spec, "null_value", None) - if isinstance(nv, float) and np.isnan(nv): - keep = ~np.isnan(raw) - else: - keep = raw != nv + keep = ~sentinel_mask(raw, nv) pos = pos[keep] raw = raw[keep] # already filtered for refinement reuse else: - col = root._schema.columns_by_name[name] - vals = root._cols[name][pos] - nv = getattr(col.spec, "null_value", None) - if isinstance(nv, float) and np.isnan(nv): - keep = ~np.isnan(vals) - else: - keep = vals != nv - pos = pos[keep] + pos = pos[~sentinel_mask(root._cols[name][pos], nv)] candidates = pos prefetched = {primary_op_name: raw} else: diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py new file mode 100644 index 000000000..8aa9f8375 --- /dev/null +++ b/src/blosc2/ctable_nulls.py @@ -0,0 +1,471 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# This source code is licensed under a BSD-style license (found in the +# LICENSE file in the root directory of this source tree) +####################################################################### + +"""Uniform access to a CTable column's validity (null) channel. + +CTable represents nulls in several different ways depending on the column +kind: an in-band **sentinel** value for fixed-width scalars and utf8, a +reserved **code** (``-1``) for dictionary columns, and native ``None`` cells +for the variable-length container kinds. A fourth representation -- a +sidecar validity array, Arrow's model -- is being added. + +:class:`NullChannel` hides that choice behind one accessor, so callers ask +*what is null* without knowing how the column stores it. Every site that +used to reach for ``getattr(spec, "null_value", None)`` and hand-roll a +comparison should go through here instead. +""" + +from __future__ import annotations + +import ast +import math +import operator +from builtins import bool as builtin_bool +from builtins import bytes as builtin_bytes + +import numpy as np + +import blosc2 +from blosc2.schema import ( + DictionarySpec, + ObjectSpec, + StructSpec, + VLBytesSpec, + VLStringSpec, +) + +#: The column stores no nulls at all. +NULL_NONE = "none" +#: Nullity lives in a sidecar ``.notnull`` validity array (Arrow's model). +NULL_MASK = "mask" +#: Nullity is an in-band sentinel value taken out of the dtype's range. +NULL_SENTINEL = "sentinel" +#: Nullity is a reserved dictionary code (``DictionarySpec.null_code``). +NULL_CODE = "code" +#: Nullity is a native ``None`` cell in a variable-length container. +NULL_NATIVE = "native" + +# Specs whose cells can hold a native ``None``. ``ListSpec`` is deliberately +# absent: it matches ``Column.is_varlen_scalar``, which is what the null API +# has always keyed off, and list columns have never participated in +# ``dropna``'s default subset. +_NATIVE_NULL_SPECS = (VLStringSpec, VLBytesSpec, StructSpec, ObjectSpec) + + +def kind_of_spec(spec) -> str: + """Return the ``NULL_*`` representation *spec* uses for its nulls. + + Dictionary and native-``None`` kinds report a null channel regardless of + their ``nullable`` flag, because their storage can represent a null + either way; for the sentinel kinds it is the presence of a + ``null_value`` that decides. + """ + if spec is None: + return NULL_NONE + if isinstance(spec, DictionarySpec): + return NULL_CODE + if isinstance(spec, _NATIVE_NULL_SPECS): + return NULL_NATIVE + # UTF8Spec is a variable-length kind but represents nulls with a sentinel + # string, so it falls through to the sentinel test below. + if getattr(spec, "null_value", None) is not None: + return NULL_SENTINEL + return NULL_NONE + + +def is_nan_sentinel(value) -> bool: + """True when *value* is a NaN used as a null sentinel. + + Accepts any NumPy float width, not just Python ``float`` -- a ``float32`` + NaN sentinel compares unequal to itself just as a Python one does, so + treating it as an ordinary value would silently stop marking nulls. + """ + return isinstance(value, (float, np.floating)) and math.isnan(value) + + +# Internal short alias; ``is_nan_sentinel`` is the name other modules import. +_is_nan = is_nan_sentinel + + +def sentinel_mask(arr: np.ndarray, null_value, *, item_ndim: int = 0) -> np.ndarray: + """Return a boolean array, True where *arr* holds *null_value*. + + The result always has one entry per *row*: for a fixed-shape ndarray + column (*item_ndim* > 0) a row counts as null only when every element of + its item equals the sentinel. + + Returns an all-False array when *null_value* is ``None``. + """ + if null_value is None: + # Before np.asarray: a ragged list column would not survive the + # conversion, and it has no sentinel to compare against anyway. + return np.zeros(len(arr), dtype=np.bool_) + arr = np.asarray(arr) + if item_ndim: + if arr.ndim <= item_ndim: + arr = arr.reshape((1, *arr.shape)) + elem_mask = np.isnan(arr) if _is_nan(null_value) else arr == null_value + inner_axes = tuple(range(1, elem_mask.ndim)) + return elem_mask.all(axis=inner_axes) if inner_axes else elem_mask.astype(np.bool_) + if np.issubdtype(arr.dtype, np.datetime64): + # Timestamp columns materialize with the int64 sentinel already decoded + # into np.datetime64('NaT') (they share the same bit pattern), so the + # sentinel value itself never appears in arr. + return np.isnat(arr) + if _is_nan(null_value): + return np.isnan(arr) + return arr == null_value + + +def is_null_value(val, null_value) -> bool: + """Scalar counterpart of :func:`sentinel_mask` for a single Python value. + + A column with no sentinel has no in-band null, so this is ``False`` there + -- native ``None`` cells are the other kinds' business, not this one's. + """ + if null_value is None: + return False + try: + if _is_nan(null_value): + return isinstance(val, (float, np.floating)) and math.isnan(val) + except TypeError: + pass + return val == null_value + + +# --------------------------------------------------------------------------- +# Null-aware predicate rewriting +# --------------------------------------------------------------------------- + + +def _collect_names(node: ast.AST) -> set[str]: + """The operand names *node* references.""" + out: set[str] = set() + _predicate_names(node, out) + return out + + +def _predicate_names(node: ast.AST, out: set[str]) -> None: + """Collect the operand names *node* references, keeping dotted paths whole.""" + if isinstance(node, (ast.Name, ast.Attribute)): + # Stop here: descending into an Attribute would also yield the bare + # prefix (``trip`` for ``trip.begin.lon``), which is not an operand. + out.add(ast.unparse(node)) + return + for child in ast.iter_child_nodes(node): + _predicate_names(child, out) + + +_COMPARE_OPS = { + ast.Lt: operator.lt, + ast.LtE: operator.le, + ast.Gt: operator.gt, + ast.GtE: operator.ge, + ast.Eq: operator.eq, + ast.NotEq: operator.ne, +} + + +def _sentinel_can_match(node: ast.Compare, name: str, sentinel) -> bool: + """Whether *sentinel* could satisfy this comparison, so a guard is needed. + + A guard only changes the answer if the sentinel would otherwise pass the + leaf. ``score > 100`` with ``null_value=-1`` needs none: ``-1`` fails the + comparison already. Skipping it there matters for more than tidiness -- + an unguarded single-predicate expression is the shape the index planner + recognizes, so a needless guard would push a query that is correct today + off its index and onto a full scan. + + Answers True (guard) whenever the comparison is not a simple + ``column literal``, since anything else cannot be settled here. + """ + if len(node.ops) != 1 or len(node.comparators) != 1: + return True + op = _COMPARE_OPS.get(type(node.ops[0])) + if op is None: + return True + left, right = node.left, node.comparators[0] + if isinstance(left, (ast.Name, ast.Attribute)) and ast.unparse(left) == name: + column_first, other = True, right + elif isinstance(right, (ast.Name, ast.Attribute)) and ast.unparse(right) == name: + column_first, other = False, left + else: + return True + try: + # literal_eval, not Constant: a negative literal such as ``-1`` is a + # UnaryOp(USub) in the tree, not a plain constant. + literal = ast.literal_eval(other) + return builtin_bool(op(sentinel, literal) if column_first else op(literal, sentinel)) + except Exception: + return True + + +class _NullPredicateRewriter(ast.NodeTransformer): + """Conjoin a validity guard onto every predicate over a nullable column.""" + + def __init__(self, guards: dict[str, tuple[str, object]]) -> None: + self._guards = guards + self._in_negation = 0 + self.changed = False + + def _guard(self, node: ast.AST, names: set[str]) -> ast.AST: + for guard in sorted({self._guards[n][0] for n in names if n in self._guards}): + # Re-parse per insertion: the same guard can appear at several + # leaves, and an AST node must not be shared between them. + right = ast.parse(guard, mode="eval").body + node = ast.BinOp(left=node, op=ast.BitAnd(), right=right) + self.changed = True + return node + + def visit_Compare(self, node: ast.Compare) -> ast.AST: + if self._in_negation: + # The enclosing negation already conjoins validity for every column + # in its subtree, and that outer conjunct dominates whatever this + # leaf evaluates to for a null row. Guarding here as well would be + # redundant, and it would put the guard on the wrong side of the + # ``~`` (see the note in :func:`rewrite_null_predicates`). It would + # also be wrong to skip it via _sentinel_can_match: a sentinel that + # fails ``a > 10`` *passes* ``not (a > 10)``. + return node + names = { + n + for n in _collect_names(node) + if n in self._guards and _sentinel_can_match(node, n, self._guards[n][1]) + } + return self._guard(node, names) + + def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: + if not isinstance(node.op, (ast.Not, ast.Invert)): + return self.generic_visit(node) + names = _collect_names(node.operand) + self._in_negation += 1 + node = self.generic_visit(node) + self._in_negation -= 1 + return self._guard(node, names) + + def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: + """Normalize ``and``/``or`` to ``&``/``|``. + + The rewrite emits ``&`` conjuncts, and leaving a mix of Python boolean + operators and bitwise ones in one expression makes the result depend on + how the downstream parser reconciles their very different precedences. + Emitting one form removes the question. + """ + node = self.generic_visit(node) + op = ast.BitAnd() if isinstance(node.op, ast.And) else ast.BitOr() + combined = node.values[0] + for value in node.values[1:]: + combined = ast.BinOp(left=combined, op=op, right=value) + return combined + + +def sentinel_guard_expr(name: str, null_value) -> str | None: + """Source text for a predicate that is True where *name* is not null. + + Emitted inline rather than injected as a precomputed boolean operand, so + the guard stays a predicate *on the same column* -- an index that can serve + ``a > 90`` can serve ``(a > 90) & (a != 999)`` too, where an opaque extra + operand would push the planner off the index and onto a full scan. + + Returns ``None`` for a sentinel that has no literal form, leaving the + caller to fall back to an injected operand. + """ + if is_nan_sentinel(null_value): + # NaN is the only value that compares unequal to itself, so this needs + # no isnan() -- and no ``~``, which would fight the negation rule below. + return f"({name} == {name})" + if isinstance(null_value, np.generic): + null_value = null_value.item() + if isinstance(null_value, (builtin_bool, int, float, str, builtin_bytes)): + return f"({name} != {null_value!r})" + return None + + +def rewrite_null_predicates(expr: str, guards: dict[str, tuple[str, object]]) -> str | None: + """Make each predicate over a nullable column reject that column's nulls. + + *guards* maps a column name, as it appears in *expr*, to a + ``(guard_text, sentinel)`` pair: the source of a predicate that is True + where that column is not null (see :func:`sentinel_guard_expr`), and the + sentinel value itself. Every comparison referencing such a column is + rewritten from ``a > 10`` into ``(a > 10) & (a != 999)``, which is SQL + ``WHERE`` semantics: a null operand satisfies no comparison. A leaf the + sentinel could never have satisfied is left alone -- see + :func:`_sentinel_can_match`. + + Doing this **per leaf** rather than once over the whole expression is what + makes ``OR`` correct. A global ``result & valid_a`` would drop a row that + is null in ``a`` but matches the other branch of ``(a > 10) | (b == 0)``, + where SQL says the row qualifies. + + Under a negation the guard is attached at the negation itself, not inside + it: ``not (a > 10)`` must be False for a null ``a``, but ``~((a > 10) & + guard)`` would yield True. Guarding outside -- ``(~(a > 10)) & guard`` -- + gives False. This is exact for every form the tests cover, and + conservative in one three-valued corner: ``not (a > 10 and b == 999)`` + where the second term is False makes SQL's ``NULL AND FALSE`` collapse to + ``FALSE``, so the row should survive the negation, while the guard drops + it. Rows are only ever dropped, never wrongly returned. + + Returns the rewritten expression, or ``None`` when nothing was rewritten -- + including when *expr* does not parse, which leaves the caller's original + text untouched for the downstream parser to report on. + """ + if not guards: + return None + try: + tree = ast.parse(expr, mode="eval") + except SyntaxError: + return None + rewriter = _NullPredicateRewriter(guards) + tree = rewriter.visit(tree) + if not rewriter.changed: + return None + ast.fix_missing_locations(tree) + return ast.unparse(tree) + + +class NullChannel: + """Uniform read accessor for one column's validity channel. + + Subsumes the representations CTable uses -- in-band sentinel, dictionary + null code, native ``None``, and (once it lands) a sidecar validity array + -- so callers ask *what is null* without knowing which one a column uses. + + Bound to a :class:`~blosc2.ctable.Column`, so it sees that column's view + (sorted order, row filter) the same way the column itself does. Nothing + is snapshotted: every property reads through to the live schema, which + keeps a cached channel correct across in-place spec mutation. + """ + + __slots__ = ("_col",) + + def __init__(self, column) -> None: + self._col = column + + def __repr__(self) -> str: + return f"NullChannel({self._col._col_name!r}, kind={self.kind!r})" + + # ------------------------------------------------------------------ + # Identity + # ------------------------------------------------------------------ + + @property + def spec(self): + """This column's schema spec, or ``None`` if it has no schema entry.""" + col_info = self._col._table._schema.columns_by_name.get(self._col._col_name) + return None if col_info is None else col_info.spec + + @property + def kind(self) -> str: + """Which ``NULL_*`` representation this column uses.""" + return kind_of_spec(self.spec) + + @property + def is_nullable(self) -> bool: + """True when this column has a null channel at all.""" + return self.kind != NULL_NONE + + @property + def sentinel(self): + """The in-band sentinel value, or ``None`` for the other kinds.""" + return getattr(self.spec, "null_value", None) + + @property + def null_code(self): + """The reserved dictionary code, or ``None`` for the other kinds.""" + return getattr(self.spec, "null_code", None) + + # ------------------------------------------------------------------ + # Reads + # ------------------------------------------------------------------ + + def mask_for_values(self, arr: np.ndarray) -> np.ndarray: + """True where an already-materialized *arr* of this column's values is null. + + Always returns one flag per row, all False when the column has no + sentinel. This is the vectorized in-band test only -- it is the right + entry point for callers that already hold the values. + """ + col = self._col + return sentinel_mask(arr, self.sentinel, item_ndim=col.item_ndim if col.is_ndarray else 0) + + def null_mask(self) -> np.ndarray: + """True where this column's live values are null, one flag per live row.""" + col = self._col + kind = self.kind + if kind == NULL_CODE: + return col._dictionary_eq(None) + if kind == NULL_NATIVE: + return np.array([v is None for v in col], dtype=np.bool_) + return self.mask_for_values(col[:]) + + def null_count(self) -> int: + """Number of live rows that are null; ``0`` in O(1) when never null.""" + kind = self.kind + if kind == NULL_NONE: + return 0 + if kind == NULL_NATIVE: + return sum(1 for v in self._col if v is None) + return int(self.null_mask().sum()) + + def nonnull_chunks(self): + """Yield chunks of live values with the null ones removed.""" + col = self._col + sentinel = self.sentinel + if sentinel is None: + yield from col.iter_chunks() + return + is_nan_sentinel = _is_nan(sentinel) + for chunk in col.iter_chunks(): + mask = ~np.isnan(chunk) if is_nan_sentinel else chunk != sentinel + filtered = chunk[mask] + if len(filtered) > 0: + yield filtered + + # ------------------------------------------------------------------ + # Lazy predicates over the raw physical array + # ------------------------------------------------------------------ + + def null_pred(self): + """Lazy predicate over the raw physical array, True where the value is null. + + Returns ``None`` when there is nothing to propagate -- the expression + layer reads that as "never null" and skips the operand entirely. + + Fixed-shape ndarray columns return ``None`` as well: their per-item + sentinel mask does not align 1:1 with the row-level predicates built + here. Use :meth:`null_mask` for those instead. Dictionary and + variable-length scalar columns never reach here, because + ``Column._ensure_queryable`` rejects them for arithmetic and + comparisons before any predicate is built. + """ + col = self._col + if col.is_ndarray: + return None + sentinel = self.sentinel + if sentinel is None: + return None + if _is_nan(sentinel): + return blosc2.isnan(col._raw_col) + return col._raw_col == sentinel + + def valid_pred(self): + """Lazy predicate over the raw physical array, True where the value is *not* null. + + Returns ``None`` under the same conditions as :meth:`null_pred`. + """ + col = self._col + if col.is_ndarray: + return None + sentinel = self.sentinel + if sentinel is None: + return None + if _is_nan(sentinel): + return ~blosc2.isnan(col._raw_col) + return col._raw_col != sentinel diff --git a/src/blosc2/groupby.py b/src/blosc2/groupby.py index 4eb1646dd..d3ec84d6c 100644 --- a/src/blosc2/groupby.py +++ b/src/blosc2/groupby.py @@ -22,6 +22,7 @@ import numpy as np +from blosc2.ctable_nulls import is_nan_sentinel, sentinel_mask from blosc2.dsl_kernel import DSLKernel from blosc2.schema import DictionarySpec, NDArraySpec, SchemaSpec, float64, int64 from blosc2.schema import bool as b2_bool @@ -598,7 +599,7 @@ def _try_execute_cython_two_int_key_hash(self, specs: list[_AggSpec]): # noqa: if value_dtype is None or np.dtype(value_dtype).kind != "f": return None null_value = getattr(value_info.spec, "null_value", None) - if null_value is not None and not (isinstance(null_value, float) and math.isnan(null_value)): + if null_value is not None and not is_nan_sentinel(null_value): return None try: @@ -749,7 +750,7 @@ def _try_execute_cython_dense_int_key(self, specs: list[_AggSpec]): # noqa: C90 desc.update({"kernel": kernel, "state_kind": "counts", "value_dtype": value_dtype}) elif spec.op in {"sum", "mean", "min", "max"}: if value_dtype.kind == "f": - skip_nan = isinstance(null_value, float) and math.isnan(null_value) + skip_nan = is_nan_sentinel(null_value) if null_value is not None and not skip_nan: return None suffix = "sum" if spec.op == "sum" else spec.op @@ -1108,7 +1109,7 @@ def _try_execute_cython_float_hash(self, specs: list[_AggSpec]): # noqa: C901 if value_dtype is None or np.dtype(value_dtype).kind != "f": return None null_value = getattr(value_info.spec, "null_value", None) - nullable_nan_value = isinstance(null_value, float) and math.isnan(null_value) + nullable_nan_value = is_nan_sentinel(null_value) if null_value is not None and not nullable_nan_value: return None @@ -2116,24 +2117,23 @@ def _result_spec_for_agg(self, spec: _AggSpec) -> SchemaSpec: def _null_mask(self, name: str, values: np.ndarray, *, is_key: bool) -> np.ndarray: col_info = self.table._schema.columns_by_name[name] spec = col_info.spec + null_value = getattr(spec, "null_value", None) if isinstance(values, _Utf8KeyChunk): - null_value = getattr(spec, "null_value", None) if null_value is None: return np.zeros(len(values), dtype=bool) return values.codes == values.code_of(null_value) if isinstance(spec, DictionarySpec): mask = values == np.int32(spec.null_code) return mask if is_key or getattr(spec, "nullable", False) else np.zeros(len(values), dtype=bool) - null_value = getattr(spec, "null_value", None) mask = np.zeros(len(values), dtype=bool) # For keys, treat all NaNs as missing so dropna behaves predictably. # For values, only nullable NaN sentinels are skipped. - if values.dtype.kind == "f" and ( - is_key or (isinstance(null_value, float) and math.isnan(null_value)) - ): + nan_sentinel = is_nan_sentinel(null_value) + if values.dtype.kind == "f" and (is_key or nan_sentinel): mask |= np.isnan(values) - if null_value is not None and not (isinstance(null_value, float) and math.isnan(null_value)): - mask |= values == null_value + if null_value is not None and not nan_sentinel: + # A float key column with a non-NaN sentinel gets both tests. + mask |= sentinel_mask(values, null_value) return mask diff --git a/src/blosc2/schema_validation.py b/src/blosc2/schema_validation.py index faa7dd2a7..db8ff9390 100644 --- a/src/blosc2/schema_validation.py +++ b/src/blosc2/schema_validation.py @@ -14,13 +14,14 @@ from __future__ import annotations -import math from dataclasses import MISSING from typing import Any import numpy as np from pydantic import BaseModel, Field, ValidationError, create_model +from blosc2.ctable_nulls import is_null_value as _null_value_matches +from blosc2.ctable_nulls import sentinel_mask from blosc2.list_array import _coerce_struct_item, coerce_list_cell from blosc2.schema import ListSpec, NDArraySpec, StructSpec from blosc2.schema_compiler import CompiledSchema # noqa: TC001 @@ -71,16 +72,7 @@ def build_validator_model(schema: CompiledSchema) -> type[BaseModel]: def _is_null_value(val, null_value) -> bool: """Return True if *val* equals the null sentinel, handling NaN correctly.""" - import math - - if null_value is None: - return False - try: - if isinstance(null_value, (float, np.floating)) and math.isnan(null_value): - return isinstance(val, (float, np.floating)) and math.isnan(val) - except TypeError: - pass - return val == null_value + return _null_value_matches(val, null_value) def _mask_nulls(schema: CompiledSchema, row: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: @@ -104,9 +96,7 @@ def _mask_nulls(schema: CompiledSchema, row: dict[str, Any]) -> tuple[dict[str, try: arr = np.asarray(val, dtype=col.spec.dtype) is_null = arr.shape == col.spec.item_shape and bool( - np.isnan(arr).all() - if isinstance(nv, (float, np.floating)) and math.isnan(nv) - else (arr == nv).all() + sentinel_mask(arr, nv, item_ndim=len(col.spec.item_shape))[0] ) except Exception: is_null = val is None diff --git a/src/blosc2/schema_vectorized.py b/src/blosc2/schema_vectorized.py index e08736eb8..5689246d4 100644 --- a/src/blosc2/schema_vectorized.py +++ b/src/blosc2/schema_vectorized.py @@ -19,6 +19,7 @@ import numpy as np +from blosc2.ctable_nulls import sentinel_mask from blosc2.list_array import _coerce_struct_item, coerce_list_cell from blosc2.schema import ListSpec, NDArraySpec, ObjectSpec, StructSpec from blosc2.schema_compiler import CompiledColumn, CompiledSchema # noqa: TC001 @@ -51,14 +52,7 @@ def _null_mask_for_spec(arr: np.ndarray, spec) -> np.ndarray | None: null_value = getattr(spec, "null_value", None) if null_value is None: return None - try: - import math - - if isinstance(null_value, float) and math.isnan(null_value): - return np.isnan(arr) - except TypeError: - pass - return arr == null_value + return sentinel_mask(arr, null_value) def validate_column_values(col: CompiledColumn, values: Any) -> None: # noqa: C901 diff --git a/tests/ctable/test_null_channel.py b/tests/ctable/test_null_channel.py new file mode 100644 index 000000000..b58aee1f1 --- /dev/null +++ b/tests/ctable/test_null_channel.py @@ -0,0 +1,267 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Tests for the NullChannel abstraction (``blosc2.ctable_nulls``). + +Phase 0 of the mask-based-nulls plan: the channel unifies how a column's +nullity is read, without changing any observable behavior. These tests pin +the classification and the shared sentinel helpers so later phases (which add +a ``"mask"`` kind) have something to diff against. +""" + +from __future__ import annotations + +import dataclasses +from dataclasses import dataclass + +import numpy as np +import pytest + +import blosc2 +from blosc2 import CTable +from blosc2.ctable_nulls import ( + NULL_CODE, + NULL_NATIVE, + NULL_NONE, + NULL_SENTINEL, + is_nan_sentinel, + is_null_value, + kind_of_spec, + sentinel_mask, +) + +# --------------------------------------------------------------------------- +# kind classification +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("spec", "expected"), + [ + (blosc2.int64(), NULL_NONE), + (blosc2.int64(null_value=-1), NULL_SENTINEL), + (blosc2.float64(null_value=float("nan")), NULL_SENTINEL), + (blosc2.string(max_length=8), NULL_NONE), + (blosc2.string(max_length=8, null_value=""), NULL_SENTINEL), + (blosc2.bytes(max_length=8, null_value=b""), NULL_SENTINEL), + (blosc2.timestamp(null_value=-1), NULL_SENTINEL), + # bool(nullable=True) has no sentinel until the schema is compiled: + # _resolve_nullable_specs is what picks 255. See + # test_kind_of_spec_resolves_on_compile. + (blosc2.bool(nullable=True), NULL_NONE), + (blosc2.bool(nullable=True, null_value=255), NULL_SENTINEL), + (blosc2.bool(), NULL_NONE), + # utf8 is a variable-length kind but stores nulls as a sentinel string. + (blosc2.utf8(), NULL_NONE), + (blosc2.utf8(null_value="__NULL__"), NULL_SENTINEL), + # Dictionary and native-None kinds report a channel either way: their + # storage can represent a null regardless of the nullable flag. + (blosc2.dictionary(), NULL_CODE), + (blosc2.dictionary(nullable=True), NULL_CODE), + (blosc2.vlstring(), NULL_NATIVE), + (blosc2.vlbytes(), NULL_NATIVE), + ], +) +def test_kind_of_spec(spec, expected): + assert kind_of_spec(spec) == expected + + +def test_kind_of_spec_none(): + assert kind_of_spec(None) == NULL_NONE + + +def test_kind_of_spec_resolves_on_compile(): + """``nullable=True`` only becomes a sentinel once the table compiles it.""" + + @dataclass + class Row: + flag: bool = blosc2.field(blosc2.bool(nullable=True)) + + t = CTable(Row) + t.append({"flag": True}) + assert t["flag"].null_storage == NULL_SENTINEL + assert t["flag"].null_value == 255 + + +def test_null_storage_property_matches_spec(): + @dataclass + class Row: + plain: int = blosc2.field(blosc2.int64()) + nulled: int = blosc2.field(blosc2.int64(null_value=-1)) + tag: str = blosc2.field(blosc2.dictionary()) + note: str = blosc2.field(blosc2.vlstring()) + + t = CTable(Row) + t.append({"plain": 1, "nulled": 2, "tag": "a", "note": "n"}) + assert t["plain"].null_storage == NULL_NONE + assert t["nulled"].null_storage == NULL_SENTINEL + assert t["tag"].null_storage == NULL_CODE + assert t["note"].null_storage == NULL_NATIVE + + +def test_channel_reads_through_to_live_schema(): + """A cached channel must not snapshot the sentinel. + + ``_resolve_nullable_specs`` assigns ``spec.null_value`` in place, so a + channel that copied it at construction time would report the wrong kind. + """ + + @dataclass + class Row: + v: int = blosc2.field(blosc2.int64()) + + t = CTable(Row) + t.append({"v": 1}) + col = t["v"] + channel = col._nulls + assert channel.kind == NULL_NONE + + t._schema.columns_by_name["v"].spec.null_value = -1 + assert channel.kind == NULL_SENTINEL + assert channel.sentinel == -1 + + +# --------------------------------------------------------------------------- +# sentinel_mask +# --------------------------------------------------------------------------- + + +def test_sentinel_mask_plain_value(): + arr = np.array([1, -1, 3, -1]) + np.testing.assert_array_equal(sentinel_mask(arr, -1), [False, True, False, True]) + + +def test_sentinel_mask_none_is_all_false(): + arr = np.array([1, 2, 3]) + np.testing.assert_array_equal(sentinel_mask(arr, None), [False, False, False]) + + +def test_sentinel_mask_none_does_not_coerce_ragged_input(): + """A list column reaches here with ragged rows; asarray would raise.""" + ragged = [[1, 2], [3], []] + np.testing.assert_array_equal(sentinel_mask(ragged, None), [False, False, False]) + + +def test_sentinel_mask_nan_sentinel(): + arr = np.array([1.0, np.nan, 3.0]) + np.testing.assert_array_equal(sentinel_mask(arr, float("nan")), [False, True, False]) + + +def test_sentinel_mask_nan_sentinel_narrow_float(): + """A float32 NaN sentinel must still be recognized as NaN, not compared.""" + arr = np.array([1.0, np.nan, 3.0], dtype=np.float32) + np.testing.assert_array_equal(sentinel_mask(arr, np.float32("nan")), [False, True, False]) + + +def test_sentinel_mask_datetime_uses_nat(): + """Timestamp values arrive already decoded to NaT, not as the raw sentinel.""" + arr = np.array(["2020-01-01", "NaT", "2021-01-01"], dtype="datetime64[s]") + np.testing.assert_array_equal(sentinel_mask(arr, np.iinfo(np.int64).min), [False, True, False]) + + +def test_sentinel_mask_ndarray_needs_every_element(): + """An ndarray row is null only when the whole item is the sentinel.""" + arr = np.array([[0, 0], [0, 5], [9, 9]]) + np.testing.assert_array_equal(sentinel_mask(arr, 0, item_ndim=1), [True, False, False]) + + +def test_sentinel_mask_ndarray_promotes_bare_row(): + arr = np.array([0, 0]) + np.testing.assert_array_equal(sentinel_mask(arr, 0, item_ndim=1), [True]) + + +# --------------------------------------------------------------------------- +# scalar helpers +# --------------------------------------------------------------------------- + + +def test_is_nan_sentinel(): + assert is_nan_sentinel(float("nan")) + assert is_nan_sentinel(np.float32("nan")) + assert is_nan_sentinel(np.float64("nan")) + assert not is_nan_sentinel(0.0) + assert not is_nan_sentinel(None) + assert not is_nan_sentinel("nan") + + +def test_is_null_value(): + assert is_null_value(-1, -1) + assert not is_null_value(0, -1) + assert is_null_value(float("nan"), float("nan")) + assert not is_null_value(1.0, float("nan")) + # No sentinel means no in-band null; native None cells are another kind. + assert not is_null_value(None, None) + + +# --------------------------------------------------------------------------- +# channel reads agree with the public Column API +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("spec", "annotation", "values", "null"), + [ + (blosc2.int64(null_value=-1), int, [1, 2, 3], -1), + (blosc2.float64(null_value=float("nan")), float, [1.0, 2.0, 3.0], float("nan")), + (blosc2.string(max_length=8, null_value=""), str, ["a", "b", "c"], ""), + (blosc2.utf8(null_value="__NULL__"), str, ["a", "b", "c"], "__NULL__"), + ], +) +def test_channel_agrees_with_column_api(spec, annotation, values, null): + Row = dataclasses.make_dataclass("Row", [("v", annotation, blosc2.field(spec))]) + + t = CTable(Row) + t.extend({"v": [*values, null]}) + col = t["v"] + + np.testing.assert_array_equal(col._nulls.null_mask(), col.is_null()) + assert col._nulls.null_count() == col.null_count() == 1 + assert col._nulls.is_nullable + assert col._nulls.sentinel == null or is_nan_sentinel(null) + + nonnull = np.concatenate(list(col._nulls.nonnull_chunks())) + assert len(nonnull) == len(values) + + +def test_null_pred_is_none_for_non_nullable(): + @dataclass + class Row: + v: int = blosc2.field(blosc2.int64()) + + t = CTable(Row) + t.extend({"v": [1, 2, 3]}) + assert t["v"]._nulls.null_pred() is None + assert t["v"]._nulls.valid_pred() is None + + +def test_null_pred_is_none_for_ndarray_column(): + """Per-item sentinel masks do not align 1:1 with row-level predicates.""" + + @dataclass + class Row: + v: object = blosc2.field(blosc2.ndarray((2,), dtype=blosc2.int64(), null_value=-1)) + + t = CTable(Row) + t.extend({"v": np.array([[1, 2], [-1, -1]])}) + col = t["v"] + assert col._nulls.null_pred() is None + # is_null() still works: it reduces per item. + np.testing.assert_array_equal(col.is_null(), [False, True]) + + +def test_null_pred_matches_is_null_for_sentinel_column(): + @dataclass + class Row: + v: int = blosc2.field(blosc2.int64(null_value=-1)) + + t = CTable(Row) + t.extend({"v": [1, -1, 3]}) + col = t["v"] + pred = np.asarray(col._nulls.null_pred().compute()[:]) + np.testing.assert_array_equal(pred[: t.nrows], col.is_null()) + valid = np.asarray(col._nulls.valid_pred().compute()[:]) + np.testing.assert_array_equal(valid[: t.nrows], col.notnull()) diff --git a/tests/ctable/test_null_predicate_rewrite.py b/tests/ctable/test_null_predicate_rewrite.py new file mode 100644 index 000000000..b466d23fb --- /dev/null +++ b/tests/ctable/test_null_predicate_rewrite.py @@ -0,0 +1,246 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""String predicates over nullable columns follow SQL ``WHERE`` semantics. + +A null operand satisfies no comparison. The operator form has always done +this (``Column._null_aware_compare``); the string form gets there through +``CTable._rewrite_null_predicates``, which conjoins a validity guard onto each +comparison leaf that reads a nullable column. + +Per *leaf*, not once over the whole expression: a global ``result & valid_a`` +would drop a row that is null in ``a`` but matches the other branch of +``(a > 10) | (b == 0)``, which SQL says qualifies. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 +from blosc2 import CTable +from blosc2.ctable_nulls import rewrite_null_predicates, sentinel_guard_expr + +# --------------------------------------------------------------------------- +# The pure string -> string transform +# --------------------------------------------------------------------------- + +# name -> (guard text, sentinel value). 999 satisfies "> 10", so every leaf +# below genuinely needs its guard. +GUARDS = {"a": ("(a != 999)", 999), "trip.lon": ("(trip.lon != 999)", 999)} + + +@pytest.mark.parametrize( + ("expr", "expected"), + [ + ("a > 10", "(a > 10) & (a != 999)"), + # Per leaf, so the OR branch that does not read `a` stays reachable. + ("(a > 10) | (b == 0)", "(a > 10) & (a != 999) | (b == 0)"), + ("a + b > 10", "(a + b > 10) & (a != 999)"), + # Comparing against the sentinel itself must match nothing. + ("a == 999", "(a == 999) & (a != 999)"), + # Nested leaves keep their dotted name; the nested rewrite runs later. + ("trip.lon > 10", "(trip.lon > 10) & (trip.lon != 999)"), + # and/or are normalized to &/| so one precedence rule governs the result. + ("a > 10 and b < 3", "(a > 10) & (a != 999) & (b < 3)"), + ("a > 10 or b < 3", "(a > 10) & (a != 999) | (b < 3)"), + ], +) +def test_rewrite_shapes(expr, expected): + assert rewrite_null_predicates(expr, GUARDS) == expected + + +def test_rewrite_guards_outside_a_negation(): + """``not (a > 10)`` must be False for a null ``a``, not True. + + Guarding inside -- ``~((a > 10) & valid)`` -- would yield True. + """ + out = rewrite_null_predicates("~(a > 10)", GUARDS) + assert out == "~(a > 10) & (a != 999)" + out = rewrite_null_predicates("not (a > 10)", GUARDS) + assert out == "(not a > 10) & (a != 999)" + + +def test_rewrite_skips_leaves_the_sentinel_cannot_satisfy(): + """A guard that cannot change the answer is not emitted. + + Beyond tidiness: an unguarded single-predicate expression is the shape the + index planner recognizes, so a needless guard would push a query that is + correct today off its index onto a full scan. + """ + guards = {"a": ("(a != -1)", -1)} + assert rewrite_null_predicates("a > 10", guards) is None + assert rewrite_null_predicates("a < 10", guards) == "(a < 10) & (a != -1)" + # ...but never inside a negation: a sentinel that fails `a > 10` passes + # `not (a > 10)`. + assert rewrite_null_predicates("~(a > 10)", guards) == "~(a > 10) & (a != -1)" + + +def test_rewrite_nan_sentinel_only_guards_not_equal(): + """Every comparison with NaN is False -- except ``!=``, which is True.""" + guards = {"a": ("(a == a)", float("nan"))} + assert rewrite_null_predicates("a > 10", guards) is None + assert rewrite_null_predicates("a == 10", guards) is None + assert rewrite_null_predicates("a != 10", guards) == "(a != 10) & (a == a)" + + +def test_rewrite_returns_none_when_nothing_applies(): + assert rewrite_null_predicates("b > 0", GUARDS) is None + assert rewrite_null_predicates("a > 10", {}) is None + assert rewrite_null_predicates("a > > 10", GUARDS) is None # unparseable + + +@pytest.mark.parametrize( + ("null_value", "expected"), + [ + (-1, "(x != -1)"), + (255, "(x != 255)"), + (0.5, "(x != 0.5)"), + ("", "(x != '')"), + (b"", "(x != b'')"), + (np.int64(-1), "(x != -1)"), + ], +) +def test_sentinel_guard_expr(null_value, expected): + assert sentinel_guard_expr("x", null_value) == expected + + +def test_sentinel_guard_expr_nan_compares_to_itself(): + """NaN is the only value unequal to itself, so this needs no isnan().""" + assert sentinel_guard_expr("x", float("nan")) == "(x == x)" + + +# --------------------------------------------------------------------------- +# End-to-end SQL semantics +# --------------------------------------------------------------------------- + + +def _table(spec, annotation, values, sentinel): + Row = dataclasses.make_dataclass( + "Row", + [("a", annotation, blosc2.field(spec)), ("b", int, blosc2.field(blosc2.int64()))], + ) + t = CTable(Row) + t.extend({"a": values, "b": np.arange(len(values))}) + return t + + +# Each case: sentinel, the four `a` values (index 2 is the null), and a probe +# value. The int-999 case is the one that is wrong before this change: the +# sentinel satisfies `a > 10`, so the null row leaked into the result. +CASES = [ + ("int-sentinel-below", blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1), + ("int-sentinel-above", blosc2.int64(null_value=999), int, [1, 20, 999, 30], 999), + ("nan", blosc2.float64(null_value=float("nan")), float, [1.0, 20.0, np.nan, 30.0], np.nan), + ("string", blosc2.string(max_length=8, null_value=""), str, ["aa", "zz", "", "mm"], ""), +] + + +@pytest.mark.parametrize(("label", "spec", "annotation", "values", "sentinel"), CASES) +def test_string_predicate_matches_sql(label, spec, annotation, values, sentinel): + t = _table(spec, annotation, values, sentinel) + arr = np.array(values) + valid = np.isnan(arr) == False if label == "nan" else arr != sentinel # noqa: E712 + b = np.arange(len(values)) + threshold = "'mm'" if label == "string" else "10" + cmp = arr > (np.array("mm") if label == "string" else 10) + + got = sorted(t.where(f"a > {threshold}")["b"][:].tolist()) + assert got == sorted(b[cmp & valid].tolist()) + + # OR: the null row must still qualify through the other branch. + got = sorted(t.where(f"(a > {threshold}) | (b == 2)")["b"][:].tolist()) + assert got == sorted(b[(cmp & valid) | (b == 2)].tolist()) + + # Negation: a null operand makes `not (...)` False, not True. + got = sorted(t.where(f"~(a > {threshold})")["b"][:].tolist()) + assert got == sorted(b[~cmp & valid].tolist()) + + +@pytest.mark.parametrize(("label", "spec", "annotation", "values", "sentinel"), CASES) +def test_string_form_agrees_with_operator_form(label, spec, annotation, values, sentinel): + """The two ways of spelling a predicate must return the same rows.""" + t = _table(spec, annotation, values, sentinel) + threshold = "mm" if label == "string" else 10 + quoted = f"'{threshold}'" if label == "string" else threshold + + assert sorted(t.where(f"a > {quoted}")["b"][:].tolist()) == sorted( + t.where(t["a"] > threshold)["b"][:].tolist() + ) + assert sorted(t.where(f"(a > {quoted}) | (b == 2)")["b"][:].tolist()) == sorted( + t.where((t["a"] > threshold) | (t["b"] == 2))["b"][:].tolist() + ) + + +def test_comparing_against_the_sentinel_matches_nothing(): + """The sentinel is not a value: SQL has no row whose `a` equals NULL.""" + t = _table(blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1) + assert t.where("a == -1").nrows == 0 + assert sorted(t.where("a != -1")["b"][:].tolist()) == [0, 1, 3] + + +def test_non_nullable_column_is_untouched(): + t = _table(blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1) + assert sorted(t.where("b > 1")["b"][:].tolist()) == [2, 3] + + +def test_dropna_then_predicate_is_consistent(): + t = _table(blosc2.int64(null_value=999), int, [1, 20, 999, 30], 999) + assert sorted(t.where("a > 10")["b"][:].tolist()) == sorted(t.dropna().where("a > 10")["b"][:].tolist()) + + +# --------------------------------------------------------------------------- +# Index and scan must agree +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sentinel", [-1.0, 999.0, float("nan")]) +@pytest.mark.parametrize( + "expr", ["a > 90", "a != 50", "(a > 90) | (b < 10)", "(a > 90) & (b < 5000)", "~(a > 90)"] +) +def test_indexed_matches_unindexed(sentinel, expr): + """An ordered index answers by sorted range and never evaluates the + predicate -- a NaN sentinel sorts last, so it lands inside every ``>`` + range. Making the expression null-aware cannot fix that, so the index path + keeps its own null exclusion; this pins the two paths together. + """ + Row = dataclasses.make_dataclass( + "Row", + [ + ("a", float, blosc2.field(blosc2.float64(null_value=sentinel))), + ("b", int, blosc2.field(blosc2.int64())), + ], + ) + n = 20_000 + rng = np.random.default_rng(0) + a = rng.integers(0, 100, n).astype(np.float64) + a[::13] = sentinel + payload = {"a": a, "b": np.arange(n)} + + plain = CTable(Row) + plain.extend(payload) + indexed = CTable(Row) + indexed.extend(payload) + indexed.create_index("a", kind=blosc2.IndexKind.FULL) + + expected = np.sort(plain.where(expr)["b"][:]) + assert np.array_equal(np.sort(indexed.where(expr)["b"][:]), expected) + + # ...and both agree with SQL. + valid = ~np.isnan(a) if np.isnan(sentinel) else a != sentinel + b = np.arange(n) + sql = { + "a > 90": (a > 90) & valid, + "a != 50": (a != 50) & valid, + "(a > 90) | (b < 10)": ((a > 90) & valid) | (b < 10), + "(a > 90) & (b < 5000)": (a > 90) & valid & (b < 5000), + "~(a > 90)": ~(a > 90) & valid, + }[expr] + assert np.array_equal(expected, np.sort(b[sql])) From fc471087ff79c0121f51a2461df6989dcadbdba6 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 08:15:01 +0200 Subject: [PATCH 02/47] Let schema specs declare where their nulls live (mask-based-nulls 2) Phase 2 of plans/mask-based-nulls.md: the declaration layer for validity-mask null storage. Nothing reads or writes a sidecar yet, and the default is still "sentinel", so every existing table and every schema this release writes is byte-identical to before. A _NullableSpecMixin carries the nullability plumbing that nine specs used to each re-declare, so `null_storage` reaches all of them through one kwarg and one call apiece. _null_metadata() never emits "sentinel": that is still the default storage, so sentinel tables serialize exactly as they did and keep opening in readers that predate masks. Only "mask" is recorded, and only a mask column raises the schema version to 3 -- schema_to_dict computes the version as an explicit feature max, so a table without one stays at 1 or 2. schema_from_dict accepts 3 and its rejection message now names the version and points at convert_nulls(to='sentinel'). Two consequences fall out for free. A nullable bool under mask storage stays a real np.bool_ column instead of uint8 with a reserved 255, and a nullable string keeps its declared width instead of being widened to fit the sentinel text. Complex columns become nullable for the first time, mask-only: there is no complex value safe to reserve, which is the argument for a side channel in miniature, so nullable=True resolves to mask there regardless of policy. CTable._resolved_null_storage is the single decision point, resolving in order: explicit spec.null_storage, explicit null_value, a per-column policy.column_null_values entry, a type-wide policy sentinel field covering this kind, then policy.null_storage. That fourth rule is what keeps existing NullPolicy(float_value=...) code working once the default flips: setting a sentinel field opts those types into sentinel storage rather than becoming silently inert. __post_init__ raises only for null_storage="mask" written alongside such a field, which is a contradiction the caller wrote down. Contrary to the plan, the bool_ -> uint8 dtype flip stays in bool.__init__ rather than moving wholly into _resolve_nullable_specs: opening a stored table rebuilds specs through spec_cls(**data) and never runs the resolver, so a persisted uint8 column has to come back as uint8 from its metadata alone. __init__ resolves what it can and the resolver corrects it in both directions once the policy has spoken. Same for NDArraySpec's bool columns. Co-Authored-By: Claude Opus 5 --- src/blosc2/ctable.py | 139 +++++++- src/blosc2/ctable_nulls.py | 34 +- src/blosc2/schema.py | 367 ++++++++++++++------ src/blosc2/schema_compiler.py | 20 +- tests/ctable/test_null_storage_schema.py | 404 +++++++++++++++++++++++ 5 files changed, 854 insertions(+), 110 deletions(-) create mode 100644 tests/ctable/test_null_storage_schema.py diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index f2159f8e4..f173419bc 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -35,6 +35,7 @@ from blosc2 import compute_chunks_blocks from blosc2.ctable_indexing import _CTableIndexingMixin from blosc2.ctable_nulls import ( + NULL_MASK, NULL_SENTINEL, NullChannel, is_nan_sentinel, @@ -170,6 +171,53 @@ class Row: unsigned_int_strategy: Literal["min", "max"] = "max" timestamp_value: int = int(np.iinfo(np.int64).min) column_null_values: Mapping[str, Any] = dataclass_field(default_factory=dict) + null_storage: Literal["mask", "sentinel"] = "sentinel" + + #: Type-wide sentinel fields, paired with the spec attribute that says + #: which columns each one covers. Setting any of them is what makes a + #: policy imply sentinel storage for those types (see __post_init__). + _SENTINEL_FIELDS: ClassVar[tuple[str, ...]] = ( + "string_value", + "bytes_value", + "float_value", + "bool_value", + "signed_int_strategy", + "unsigned_int_strategy", + "timestamp_value", + ) + + def __post_init__(self): + if self.null_storage not in ("mask", "sentinel"): + raise ValueError(f"null_storage must be 'mask' or 'sentinel', got {self.null_storage!r}") + # Setting a type-wide sentinel field alongside an explicit + # null_storage="mask" is a contradiction the caller wrote down, so say + # so. Setting one *without* an explicit null_storage is not: it simply + # means "use sentinels for the types I named", which is what + # _resolve_nullable_specs does with it. Raising on that instead would + # break existing NullPolicy(float_value=...) code on the very release + # that flips the default. + if self.null_storage == "mask": + explicit = [f for f in self._SENTINEL_FIELDS if self._sentinel_field_is_set(f)] + if explicit: + raise ValueError( + f"null_storage='mask' contradicts the type-wide sentinel field(s) " + f"{', '.join(explicit)}: a mask column has no sentinel to choose. " + f"Drop the sentinel field(s), or use column_null_values to force " + f"sentinel storage on specific columns." + ) + + def _sentinel_field_is_set(self, field_name: str) -> bool: + """True when *field_name* was given a non-default value. + + Used by :meth:`CTable._resolve_nullable_specs` to infer sentinel + storage for the types a field covers. + """ + current = getattr(self, field_name) + default = _POLICY_DEFAULTS[field_name] + if is_nan_sentinel(current) and is_nan_sentinel(default): + # float_value defaults to NaN, which never equals itself. + return False + return current != default def sentinel_for_arrow_type(self, pa, pa_type): """Return the default sentinel for *pa_type*, or ``None`` if unsupported.""" @@ -206,6 +254,13 @@ def sentinel_for_arrow_type(self, pa, pa_type): return None +#: The as-declared defaults, so a policy can tell "left alone" from +#: "explicitly set to the same value". Read from the dataclass fields rather +#: than duplicated, so it cannot drift from the declarations above. +_POLICY_DEFAULTS = { + f.name: f.default for f in dataclasses.fields(NullPolicy) if f.default is not dataclasses.MISSING +} + DEFAULT_NULL_POLICY = NullPolicy() _NULL_POLICY = contextvars.ContextVar("blosc2_null_policy", default=DEFAULT_NULL_POLICY) # Sentinel for set_printoptions params whose valid value includes ``None`` @@ -4467,6 +4522,76 @@ def _validate_null_value_for_spec(name: str, spec: SchemaSpec, null_value) -> No if isinstance(spec, b2_bytes) and not isinstance(null_value, bytes): raise TypeError(f"Null sentinel for bytes column {name!r} must be bytes") + #: Which policy sentinel field governs which spec kinds. Setting one of + #: these implies sentinel storage for the kinds it covers, so existing + #: ``NullPolicy(float_value=...)`` code keeps working once the default + #: flips to mask. + _POLICY_SENTINEL_FIELD_KINDS: ClassVar[tuple[tuple[str, tuple], ...]] = ( + ("string_value", (string, UTF8Spec)), + ("bytes_value", (b2_bytes,)), + ("bool_value", (b2_bool,)), + ("timestamp_value", (timestamp,)), + ) + + @classmethod + def _policy_implies_sentinel(cls, spec, policy) -> bool: + """True when a type-wide policy sentinel field covers *spec*'s kind.""" + for field_name, kinds in cls._POLICY_SENTINEL_FIELD_KINDS: + if isinstance(spec, kinds) and policy._sentinel_field_is_set(field_name): + return True + dtype = getattr(spec, "dtype", None) + kind = np.dtype(dtype).kind if dtype is not None else "" + if kind == "f" and policy._sentinel_field_is_set("float_value"): + return True + if kind == "i" and policy._sentinel_field_is_set("signed_int_strategy"): + return True + return bool(kind == "u" and policy._sentinel_field_is_set("unsigned_int_strategy")) + + @classmethod + def _resolved_null_storage(cls, name: str, spec, policy) -> str: + """Decide where *spec*'s nulls live. The single decision point. + + In order: an explicit ``spec.null_storage`` wins; then an explicit + ``null_value`` (which *is* a request for in-band storage); then a + per-column ``policy.column_null_values`` entry; then a type-wide + policy sentinel field covering this kind; and finally the policy's + own ``null_storage`` default. + """ + if spec.null_storage is not None: + return spec.null_storage + if getattr(spec, "null_value", None) is not None: + return NULL_SENTINEL + if name in policy.column_null_values: + return NULL_SENTINEL + if not getattr(spec, "supports_sentinel", True): + # Complex has no representable sentinel, so it is mask or nothing. + return NULL_MASK + if cls._policy_implies_sentinel(spec, policy): + return NULL_SENTINEL + return policy.null_storage + + @staticmethod + def _unflip_mask_bool_dtype(spec) -> None: + """Undo the conservative uint8 flip that ``__init__`` applies to bools. + + ``bool.__init__`` cannot know whether a bare ``nullable=True`` will + resolve to mask or sentinel, and it has to assume sentinel so that + *opening* a stored table -- which rebuilds specs without running this + resolver -- brings a persisted uint8 column back as uint8. Once the + policy has spoken for mask, the column is a real ``np.bool_`` again. + """ + if spec.dtype != np.dtype(np.uint8): + return + if isinstance(spec, b2_bool): + spec.dtype = np.dtype(np.bool_) + elif isinstance(spec, NDArraySpec): + spec.dtype = np.dtype(np.bool_) + spec.itemsize = spec.dtype.itemsize + spec.kind = spec.dtype.kind + spec.type = spec.dtype.type + spec.str = spec.dtype.str + spec.name = spec.dtype.name + @classmethod def _resolve_nullable_specs( cls, schema: CompiledSchema, *, validate_column_null_values: bool = True @@ -4501,6 +4626,16 @@ def _resolve_nullable_specs( continue if not getattr(spec, "nullable", False): continue + if cls._resolved_null_storage(col.name, spec, policy) == NULL_MASK: + # Mask storage keeps nullity out of band, so none of the + # sentinel machinery below applies: no sentinel is chosen, no + # max_length widening for string/bytes, no bool -> uint8 flip. + spec.null_storage = NULL_MASK + cls._unflip_mask_bool_dtype(spec) + col.dtype = getattr(spec, "dtype", None) + col.display_width = compute_display_width(spec) + continue + spec.null_storage = NULL_SENTINEL null_value = policy.column_null_values.get(col.name) if null_value is None: null_value = cls._policy_null_value_for_spec(spec, policy) @@ -7253,9 +7388,7 @@ def _arrow_type_to_spec( # noqa: C901 for arrow_t, spec_cls in mapping: if pa_type == arrow_t: - if null_value is not None and hasattr(spec_cls(), "null_value"): - return spec_cls(null_value=null_value) - if null_value is not None and spec_cls is b2s.bool: + if null_value is not None and getattr(spec_cls, "supports_sentinel", False): return spec_cls(null_value=null_value) return spec_cls() diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index 8aa9f8375..bbf6b3d86 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -32,23 +32,37 @@ import blosc2 from blosc2.schema import ( + NULL_CODE, + NULL_MASK, + NULL_NATIVE, + NULL_NONE, + NULL_SENTINEL, DictionarySpec, ObjectSpec, StructSpec, VLBytesSpec, VLStringSpec, + fill_value_for, ) -#: The column stores no nulls at all. -NULL_NONE = "none" -#: Nullity lives in a sidecar ``.notnull`` validity array (Arrow's model). -NULL_MASK = "mask" -#: Nullity is an in-band sentinel value taken out of the dtype's range. -NULL_SENTINEL = "sentinel" -#: Nullity is a reserved dictionary code (``DictionarySpec.null_code``). -NULL_CODE = "code" -#: Nullity is a native ``None`` cell in a variable-length container. -NULL_NATIVE = "native" +# The NULL_* constants and fill_value_for are defined in blosc2.schema -- the +# lower layer, since this module imports the spec classes from it -- and +# re-exported here, which is where the null machinery otherwise lives. +__all__ = [ + "NULL_CODE", + "NULL_MASK", + "NULL_NATIVE", + "NULL_NONE", + "NULL_SENTINEL", + "NullChannel", + "fill_value_for", + "is_nan_sentinel", + "is_null_value", + "kind_of_spec", + "rewrite_null_predicates", + "sentinel_guard_expr", + "sentinel_mask", +] # Specs whose cells can hold a native ``None``. ``ListSpec`` is deliberately # absent: it matches ``Column.is_varlen_scalar``, which is what the null API diff --git a/src/blosc2/schema.py b/src/blosc2/schema.py index bdb131ae0..ee86d0ea5 100644 --- a/src/blosc2/schema.py +++ b/src/blosc2/schema.py @@ -33,6 +33,120 @@ def _normalize_scalar_value(value): return value +# How a column represents its nulls. Defined here rather than in +# ``ctable_nulls``, which is where the null machinery otherwise lives, only +# because that module imports the spec classes from this one -- this is the +# lower layer. ``ctable_nulls`` re-exports these names. +NULL_NONE = "none" +NULL_MASK = "mask" +NULL_SENTINEL = "sentinel" +NULL_CODE = "code" +NULL_NATIVE = "native" + +#: The storages a spec may ask for explicitly. ``None`` means "let the null +#: policy decide when the schema is compiled". +_EXPLICIT_NULL_STORAGES = (NULL_MASK, NULL_SENTINEL) + + +class _NullableSpecMixin: + """Shared nullability plumbing for specs that can carry a null channel. + + ``supports_sentinel`` says whether this kind can represent nulls in band + at all. Complex is the one that cannot -- there is no value to steal from + the complex plane -- so it is mask-only. + + Every such spec declares the same three things: whether it is nullable, + which sentinel stands for its nulls, and where nullity is *stored* -- + in band as that sentinel, or in a sidecar validity array. Routing all + of them through one mixin is what lets ``null_storage`` reach nine specs + through a single kwarg and a single call each. + """ + + supports_sentinel = True + + def _init_nulls(self, *, nullable, null_value, null_storage) -> None: + if null_storage is not None and null_storage not in _EXPLICIT_NULL_STORAGES: + raise ValueError( + f"null_storage must be one of {_EXPLICIT_NULL_STORAGES} or None, got {null_storage!r}" + ) + if null_storage == NULL_MASK and null_value is not None: + raise ValueError( + "null_storage='mask' keeps nullity out of band, so it cannot be combined with an " + "explicit null_value (which is what in-band 'sentinel' storage means). " + "Drop one of the two." + ) + self.nullable = _builtin_bool(nullable or null_value is not None or null_storage is not None) + self.null_value = _normalize_scalar_value(null_value) + self.null_storage = null_storage + + @property + def uses_mask(self) -> _builtin_bool: + """True when nullity lives in a sidecar validity array.""" + return self.null_storage == NULL_MASK + + @property + def uses_sentinel(self) -> _builtin_bool: + """True when nullity is an in-band sentinel value. + + Note this is about *resolved* storage: a spec built as + ``nullable=True`` under the default policy has no sentinel yet, and + answers False until :meth:`CTable._resolve_nullable_specs` picks one. + """ + return self.null_value is not None + + def _null_metadata(self) -> dict[str, Any]: + """The nullability entries this spec contributes to its metadata dict. + + ``"sentinel"`` is deliberately never emitted: it is the original and + still the default storage, so a sentinel table serializes exactly as + it did before mask storage existed, and keeps opening in readers that + predate it. Only ``"mask"`` is recorded, and only that raises the + schema version. + """ + d: dict[str, Any] = {} + if self.nullable: + d["nullable"] = True + if self.null_value is not None: + d["null_value"] = self.null_value + if self.null_storage == NULL_MASK: + d["null_storage"] = NULL_MASK + return d + + +def fill_value_for(spec): + """The value written into a mask-backed column's null slots. + + Chosen to be *loud* where the dtype allows it -- ``NaN`` for floats, + ``int64.min`` for timestamps, which ``_maybe_decode_timestamp_values`` + already surfaces as ``NaT`` -- and the dtype's zero otherwise. + + This is **not** part of the format contract and is deliberately not + recorded in the schema: recording it would recreate the very sentinel + collisions that mask storage exists to avoid. Values under ``mask=False`` + are unobservable through the ``Column`` API, so the fill is an + implementation detail that may change. + """ + from blosc2.schema import timestamp as _timestamp # local: class defined below + + if isinstance(spec, _timestamp): + return int(np.iinfo(np.int64).min) + dtype = getattr(spec, "dtype", None) + if dtype is None: # utf8 and other dtype-less specs + return "" + dtype = np.dtype(dtype) + if dtype.kind == "f": + return float("nan") + if dtype.kind == "c": + return 0j + if dtype.kind == "U": + return "" + if dtype.kind == "S": + return b"" + if dtype.kind == "b": + return False + return dtype.type(0) + + # --------------------------------------------------------------------------- # Base spec class # --------------------------------------------------------------------------- @@ -81,23 +195,32 @@ def to_metadata_dict(self) -> dict[str, Any]: # and `_kind` as class attributes. -class _NumericSpec(SchemaSpec): - """Mixin for numeric specs that support constraints and null sentinels. +class _NumericSpec(_NullableSpecMixin, SchemaSpec): + """Mixin for numeric specs that support constraints and nulls. - ``nullable=True`` asks CTable to choose a null sentinel from the current - null policy when the schema is compiled. An explicit ``null_value`` takes - precedence. + ``nullable=True`` asks CTable to resolve the null representation from the + current null policy when the schema is compiled. An explicit + ``null_value`` (in-band sentinel) or ``null_storage`` takes precedence. """ _kind: str # set by each concrete subclass - def __init__(self, *, ge=None, gt=None, le=None, lt=None, nullable: bool = False, null_value=None): + def __init__( + self, + *, + ge=None, + gt=None, + le=None, + lt=None, + nullable: bool = False, + null_value=None, + null_storage: str | None = None, + ): self.ge = ge self.gt = gt self.le = le self.lt = lt - self.nullable = nullable or null_value is not None - self.null_value = _normalize_scalar_value(null_value) + self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) def to_pydantic_kwargs(self) -> dict[str, Any]: # null_value is not a Pydantic constraint — exclude it from Pydantic kwargs. @@ -108,12 +231,7 @@ def to_pydantic_kwargs(self) -> dict[str, Any]: } def to_metadata_dict(self) -> dict[str, Any]: - d: dict[str, Any] = {"kind": self._kind, **self.to_pydantic_kwargs()} - if self.nullable: - d["nullable"] = True - if self.null_value is not None: - d["null_value"] = self.null_value - return d + return {"kind": self._kind, **self.to_pydantic_kwargs(), **self._null_metadata()} # ── Signed integers ────────────────────────────────────────────────────────── @@ -205,39 +323,54 @@ class float64(_NumericSpec): _kind = "float64" -class complex64(SchemaSpec): - """64-bit complex number column (two 32-bit floats).""" +class _ComplexSpec(_NullableSpecMixin, SchemaSpec): + """Shared body for the complex specs. + + Complex columns gain nullability here for the first time, and only with + ``null_storage="mask"``: there is no sensible in-band sentinel to steal + from the complex plane, which is precisely the argument for a side + channel. ``nullable=True`` alone therefore resolves to mask storage + regardless of the policy's default. + """ - dtype = np.dtype(np.complex64) python_type = complex + supports_sentinel = False + _kind: str - def __init__(self): - pass + def __init__(self, *, nullable: bool = False, null_value=None, null_storage: str | None = None): + if null_value is not None: + raise ValueError( + "complex columns cannot use an in-band null sentinel: no complex value is safe to " + "reserve. Use null_storage='mask' instead." + ) + if nullable and null_storage is None: + null_storage = NULL_MASK + self._init_nulls(nullable=nullable, null_value=None, null_storage=null_storage) + if not self.uses_mask and self.nullable: + raise ValueError("complex columns support null_storage='mask' only") def to_pydantic_kwargs(self) -> dict[str, Any]: return {} def to_metadata_dict(self) -> dict[str, Any]: - return {"kind": "complex64"} + return {"kind": self._kind, **self._null_metadata()} -class complex128(SchemaSpec): - """128-bit complex number column (two 64-bit floats).""" +class complex64(_ComplexSpec): + """64-bit complex number column (two 32-bit floats).""" - dtype = np.dtype(np.complex128) - python_type = complex + dtype = np.dtype(np.complex64) + _kind = "complex64" - def __init__(self): - pass - def to_pydantic_kwargs(self) -> dict[str, Any]: - return {} +class complex128(_ComplexSpec): + """128-bit complex number column (two 64-bit floats).""" - def to_metadata_dict(self) -> dict[str, Any]: - return {"kind": "complex128"} + dtype = np.dtype(np.complex128) + _kind = "complex128" -class timestamp(SchemaSpec): +class timestamp(_NullableSpecMixin, SchemaSpec): """Timestamp column stored as signed 64-bit epoch offsets. The physical storage dtype is ``int64``. ``unit`` follows Arrow/NumPy @@ -249,14 +382,19 @@ class timestamp(SchemaSpec): python_type = _builtin_object def __init__( - self, *, unit: str = "us", timezone: str | None = None, nullable: bool = False, null_value=None + self, + *, + unit: str = "us", + timezone: str | None = None, + nullable: bool = False, + null_value=None, + null_storage: str | None = None, ): if unit not in {"s", "ms", "us", "ns"}: raise ValueError("timestamp unit must be one of: 's', 'ms', 'us', 'ns'") self.unit = unit self.timezone = timezone - self.nullable = nullable or null_value is not None - self.null_value = _normalize_scalar_value(null_value) + self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) def to_pydantic_kwargs(self) -> dict[str, Any]: return {} @@ -265,39 +403,41 @@ def to_metadata_dict(self) -> dict[str, Any]: d: dict[str, Any] = {"kind": "timestamp", "unit": self.unit} if self.timezone is not None: d["timezone"] = self.timezone - if self.nullable: - d["nullable"] = True - if self.null_value is not None: - d["null_value"] = self.null_value + d.update(self._null_metadata()) return d -class bool(SchemaSpec): +class bool(_NullableSpecMixin, SchemaSpec): """Boolean column. - Nullable bool columns use uint8 physical storage with values - ``0`` (false), ``1`` (true), and ``255`` (null). + Under sentinel storage a nullable bool column is physically ``uint8``, + with ``0`` (false), ``1`` (true) and ``255`` (null) -- the reserved + ``255`` is why raw reads and predicates need special handling. Under + mask storage it stays a real ``np.bool_`` column and nullity lives in + the sidecar, so none of that applies. """ dtype = np.dtype(np.bool_) python_type = _builtin_bool - def __init__(self, *, nullable: bool = False, null_value=None): - if null_value is not None and null_value != 255: + def __init__(self, *, nullable: bool = False, null_value=None, null_storage: str | None = None): + if null_value is not None and null_value != 255 and null_storage != NULL_MASK: raise ValueError("Nullable bool null_value must be 255") - self.nullable = nullable or null_value is not None - self.null_value = _normalize_scalar_value(null_value) - self.dtype = np.dtype(np.uint8) if self.nullable else np.dtype(np.bool_) + self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) + # Resolve the physical dtype as far as it can be resolved here. This + # must happen in __init__ and not only in _resolve_nullable_specs, + # because *opening* a stored table rebuilds specs through + # ``spec_cls(**data)`` and never runs the resolver: a nullable bool + # persisted as uint8 has to come back as uint8 from its metadata + # alone. When storage is still unresolved (plain ``nullable=True``, + # policy decides later) the resolver corrects this both ways. + self.dtype = np.dtype(np.uint8) if self.nullable and not self.uses_mask else np.dtype(np.bool_) def to_pydantic_kwargs(self) -> dict[str, Any]: return {} def to_metadata_dict(self) -> dict[str, Any]: - d: dict[str, Any] = {"kind": "bool"} - if self.nullable: - d["nullable"] = True - d["null_value"] = self.null_value - return d + return {"kind": "bool", **self._null_metadata()} # --------------------------------------------------------------------------- @@ -305,7 +445,7 @@ def to_metadata_dict(self) -> dict[str, Any]: # --------------------------------------------------------------------------- -class string(SchemaSpec): +class string(_NullableSpecMixin, SchemaSpec): """Fixed-width Unicode string column. Values longer than *max_length* are rejected at validation time (and @@ -327,19 +467,29 @@ class string(SchemaSpec): the current CTable null policy when the schema is compiled. null_value: Explicit null sentinel. Takes precedence over ``nullable=True``. + null_storage: + ``"sentinel"`` to reserve a value from the dtype's range for nulls, or + ``"mask"`` to keep nullity in a sidecar validity array so the whole + range stays usable. Defaults to the current null policy. """ python_type = str _DEFAULT_MAX_LENGTH = 32 def __init__( - self, *, min_length=None, max_length=None, pattern=None, nullable: bool = False, null_value=None + self, + *, + min_length=None, + max_length=None, + pattern=None, + nullable: bool = False, + null_value=None, + null_storage: str | None = None, ): self.min_length = min_length self.max_length = max_length if max_length is not None else self._DEFAULT_MAX_LENGTH self.pattern = pattern - self.nullable = nullable or null_value is not None - self.null_value = _normalize_scalar_value(null_value) + self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) self.dtype = np.dtype(f"U{self.max_length}") def to_pydantic_kwargs(self) -> dict[str, Any]: @@ -353,15 +503,10 @@ def to_pydantic_kwargs(self) -> dict[str, Any]: return d def to_metadata_dict(self) -> dict[str, Any]: - d: dict[str, Any] = {"kind": "string", **self.to_pydantic_kwargs()} - if self.nullable: - d["nullable"] = True - if self.null_value is not None: - d["null_value"] = self.null_value - return d + return {"kind": "string", **self.to_pydantic_kwargs(), **self._null_metadata()} -class bytes(SchemaSpec): +class bytes(_NullableSpecMixin, SchemaSpec): """Fixed-width bytes column. Parameters @@ -376,16 +521,27 @@ class bytes(SchemaSpec): the current CTable null policy when the schema is compiled. null_value: Explicit null sentinel. Takes precedence over ``nullable=True``. + null_storage: + ``"sentinel"`` to reserve a value from the dtype's range for nulls, or + ``"mask"`` to keep nullity in a sidecar validity array so the whole + range stays usable. Defaults to the current null policy. """ python_type = _builtin_bytes _DEFAULT_MAX_LENGTH = 32 - def __init__(self, *, min_length=None, max_length=None, nullable: bool = False, null_value=None): + def __init__( + self, + *, + min_length=None, + max_length=None, + nullable: bool = False, + null_value=None, + null_storage: str | None = None, + ): self.min_length = min_length self.max_length = max_length if max_length is not None else self._DEFAULT_MAX_LENGTH - self.nullable = nullable or null_value is not None - self.null_value = _normalize_scalar_value(null_value) + self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) self.dtype = np.dtype(f"S{self.max_length}") def to_pydantic_kwargs(self) -> dict[str, Any]: @@ -397,12 +553,7 @@ def to_pydantic_kwargs(self) -> dict[str, Any]: return d def to_metadata_dict(self) -> dict[str, Any]: - d: dict[str, Any] = {"kind": "bytes", **self.to_pydantic_kwargs()} - if self.nullable: - d["nullable"] = True - if self.null_value is not None: - d["null_value"] = self.null_value - return d + return {"kind": "bytes", **self.to_pydantic_kwargs(), **self._null_metadata()} # --------------------------------------------------------------------------- @@ -597,7 +748,7 @@ def to_metadata_dict(self) -> dict[str, Any]: return d -class UTF8Spec(SchemaSpec): +class UTF8Spec(_NullableSpecMixin, SchemaSpec): """Variable-length UTF-8 string column stored Arrow-style as offsets + bytes. Unlike :class:`string`, this spec does not use a fixed-width NumPy dtype: @@ -619,7 +770,13 @@ class UTF8Spec(SchemaSpec): python_type = str dtype = None - def __init__(self, *, nullable: _builtin_bool = False, null_value: str | None = None): + def __init__( + self, + *, + nullable: _builtin_bool = False, + null_value: str | None = None, + null_storage: str | None = None, + ): if null_value is not None and not isinstance(null_value, str): raise TypeError(f"utf8 null_value must be str, got {type(null_value).__name__!r}") if null_value == "\x00": @@ -633,19 +790,13 @@ def __init__(self, *, nullable: _builtin_bool = False, null_value: str | None = "match it against StringDType arrays, so nulls would go undetected. " "Use a longer sentinel (the default is '__BLOSC2_NULL__')." ) - self.nullable = nullable or null_value is not None - self.null_value = _normalize_scalar_value(null_value) + self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) def to_pydantic_kwargs(self) -> dict[str, Any]: return {} def to_metadata_dict(self) -> dict[str, Any]: - d: dict[str, Any] = {"kind": "utf8"} - if self.nullable: - d["nullable"] = True - if self.null_value is not None: - d["null_value"] = self.null_value - return d + return {"kind": "utf8", **self._null_metadata()} def display_label(self) -> str: return "utf8" @@ -771,7 +922,7 @@ def to_metadata_dict(self) -> dict[str, Any]: # --------------------------------------------------------------------------- -class NDArraySpec(SchemaSpec): +class NDArraySpec(_NullableSpecMixin, SchemaSpec): """Fixed-shape N-D array column for CTable. Each row stores a NumPy-compatible array with shape ``item_shape`` and @@ -781,7 +932,15 @@ class NDArraySpec(SchemaSpec): python_type = _builtin_object - def __init__(self, item_shape, dtype=np.float64, *, nullable: bool = False, null_value=None): + def __init__( + self, + item_shape, + dtype=np.float64, + *, + nullable: bool = False, + null_value=None, + null_storage: str | None = None, + ): if isinstance(item_shape, int): item_shape = (item_shape,) item_shape = tuple(int(s) for s in item_shape) @@ -791,9 +950,12 @@ def __init__(self, item_shape, dtype=np.float64, *, nullable: bool = False, null raise ValueError("All NDArraySpec item_shape dimensions must be positive.") self.item_shape = item_shape self.dtype = np.dtype(dtype) - self.nullable = nullable or null_value is not None - if null_value is not None: - self.null_value = _normalize_scalar_value(null_value) + self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) + if self.nullable and not self.uses_mask and self.dtype == np.dtype(np.bool_): + # Same reasoning as bool: opening a stored table rebuilds the spec + # without running the resolver, so the uint8 flip has to survive + # from metadata alone. + self.dtype = np.dtype(np.uint8) self.itemsize = self.dtype.itemsize self.kind = self.dtype.kind self.type = self.dtype.type @@ -809,19 +971,29 @@ def to_metadata_dict(self) -> dict[str, Any]: "item_shape": _builtin_list(self.item_shape), "dtype_str": self.dtype.str, } - if self.nullable: - d["nullable"] = True - if hasattr(self, "null_value"): - d["null_value"] = self.null_value + d.update(self._null_metadata()) return d def display_label(self) -> str: return f"ndarray{_builtin_list(self.item_shape)}[{self.dtype}]" -def ndarray(item_shape, dtype=np.float64, *, nullable: bool = False, null_value=None) -> NDArraySpec: +def ndarray( + item_shape, + dtype=np.float64, + *, + nullable: bool = False, + null_value=None, + null_storage: str | None = None, +) -> NDArraySpec: """Build a fixed-shape N-D array descriptor for CTable columns.""" - return NDArraySpec(item_shape=item_shape, dtype=dtype, nullable=nullable, null_value=null_value) + return NDArraySpec( + item_shape=item_shape, + dtype=dtype, + nullable=nullable, + null_value=null_value, + null_storage=null_storage, + ) def vlstring( @@ -850,7 +1022,9 @@ def vlstring( ) -def utf8(*, nullable: bool = False, null_value: str | None = None) -> UTF8Spec: +def utf8( + *, nullable: bool = False, null_value: str | None = None, null_storage: str | None = None +) -> UTF8Spec: """Build a variable-length UTF-8 string schema descriptor. Use this for high-cardinality or free-text string columns: values are @@ -880,6 +1054,11 @@ def utf8(*, nullable: bool = False, null_value: str | None = None) -> UTF8Spec: the current CTable null policy when the schema is compiled. null_value: Explicit null sentinel string. Takes precedence over ``nullable=True``. + null_storage: + ``"sentinel"`` to reserve a string value for nulls, or ``"mask"`` to + keep nullity in a sidecar validity array so that every string -- + including ``""`` and the default sentinel text -- stays an ordinary + value. Defaults to the current null policy. Examples -------- @@ -893,7 +1072,7 @@ def utf8(*, nullable: bool = False, null_value: str | None = None) -> UTF8Spec: from blosc2._utf8_array import string_dtype string_dtype() # fail early with a clear error on NumPy < 2.0 - return UTF8Spec(nullable=nullable, null_value=null_value) + return UTF8Spec(nullable=nullable, null_value=null_value, null_storage=null_storage) def object( diff --git a/src/blosc2/schema_compiler.py b/src/blosc2/schema_compiler.py index 2475474d7..fe782da0d 100644 --- a/src/blosc2/schema_compiler.py +++ b/src/blosc2/schema_compiler.py @@ -484,7 +484,17 @@ def schema_to_dict(schema: CompiledSchema) -> dict[str, Any]: entry["blocks"] = list(col.config.blocks) cols.append(entry) - schema_version = 2 if schema.metadata.get("nested") is not None else 1 + # Computed as an explicit feature max rather than a running counter, so + # each version says which feature demands it. The bump is *conditional on + # a mask column existing*: sentinel tables keep serializing as version 1/2 + # and stay readable by everything that came before mask storage. + uses_mask = any(getattr(col.spec, "null_storage", None) == "mask" for col in schema.columns) + if uses_mask: + schema_version = 3 + elif schema.metadata.get("nested") is not None: + schema_version = 2 + else: + schema_version = 1 result = { "version": schema_version, "columns": cols, @@ -507,8 +517,12 @@ def schema_from_dict(data: dict[str, Any]) -> CompiledSchema: If *data* uses an unknown schema version or an unknown column kind. """ version = data.get("version", 1) - if version not in (1, 2): - raise ValueError(f"Unsupported schema version {version!r}") + if version not in (1, 2, 3): + raise ValueError( + f"Unsupported schema version {version!r}. Version 3 is written only when a column uses " + f"validity-mask null storage; an older reader can be given a readable copy with " + f"table.convert_nulls(to='sentinel')." + ) columns: list[CompiledColumn] = [] for entry in data["columns"]: diff --git a/tests/ctable/test_null_storage_schema.py b/tests/ctable/test_null_storage_schema.py new file mode 100644 index 000000000..ba0e1764e --- /dev/null +++ b/tests/ctable/test_null_storage_schema.py @@ -0,0 +1,404 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Schema plumbing for validity-mask null storage (Phase 2). + +Specs can now say *where* their nulls live -- in band as a sentinel, or in a +sidecar validity array -- and the schema records it. Nothing reads or writes +a sidecar yet; this is the declaration layer. + +The default is still ``"sentinel"``, so every existing table and every schema +written by this release is byte-identical to before. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 +from blosc2 import CTable +from blosc2.schema import fill_value_for +from blosc2.schema_compiler import schema_from_dict, schema_to_dict + +# --------------------------------------------------------------------------- +# Spec-level declaration +# --------------------------------------------------------------------------- + +MASKABLE_SPECS = [ + ("int8", blosc2.int8, {}), + ("int64", blosc2.int64, {}), + ("uint8", blosc2.uint8, {}), + ("float64", blosc2.float64, {}), + ("bool", blosc2.bool, {}), + ("timestamp", blosc2.timestamp, {}), + ("string", blosc2.string, {"max_length": 4}), + ("bytes", blosc2.bytes, {"max_length": 4}), + ("utf8", blosc2.utf8, {}), +] + + +@pytest.mark.parametrize(("label", "factory", "kwargs"), MASKABLE_SPECS) +def test_spec_accepts_null_storage(label, factory, kwargs): + spec = factory(null_storage="mask", **kwargs) + assert spec.null_storage == "mask" + assert spec.uses_mask + assert spec.nullable + assert spec.null_value is None + + +@pytest.mark.parametrize(("label", "factory", "kwargs"), MASKABLE_SPECS) +def test_spec_defaults_to_unresolved_storage(label, factory, kwargs): + """``nullable=True`` alone defers the choice to the policy.""" + assert factory(nullable=True, **kwargs).null_storage is None + assert factory(**kwargs).null_storage is None + + +def test_mask_and_sentinel_together_is_rejected(): + with pytest.raises(ValueError, match="cannot be combined with an explicit null_value"): + blosc2.int64(null_storage="mask", null_value=-1) + + +def test_unknown_null_storage_is_rejected(): + with pytest.raises(ValueError, match="null_storage must be one of"): + blosc2.int64(null_storage="bitmap") + + +def test_ndarray_spec_accepts_null_storage(): + spec = blosc2.ndarray((2,), dtype=blosc2.int64(), null_storage="mask") + assert spec.uses_mask + assert spec.null_value is None + + +# --------------------------------------------------------------------------- +# Nullable bool loses the 255 reservation under mask storage +# --------------------------------------------------------------------------- + + +def test_sentinel_bool_is_still_uint8(): + spec = blosc2.bool(nullable=True, null_value=255) + assert spec.dtype == np.dtype(np.uint8) + + +def test_mask_bool_stays_bool(): + """The point of the design: no reserved 255, no uint8 leak.""" + assert blosc2.bool(null_storage="mask").dtype == np.dtype(np.bool_) + + +def test_mask_bool_accepts_any_null_value_rejection(): + with pytest.raises(ValueError, match="Nullable bool null_value must be 255"): + blosc2.bool(nullable=True, null_value=7) + + +def test_stored_uint8_bool_reopens_as_uint8_without_the_resolver(): + """Opening a table rebuilds specs through ``spec_cls(**data)`` and never + runs ``_resolve_nullable_specs``, so the uint8 flip has to survive from + metadata alone. + """ + spec = blosc2.bool(nullable=True, null_value=255) + rebuilt = schema_from_dict({"version": 1, "columns": [{"name": "flag", **spec.to_metadata_dict()}]}) + assert rebuilt.columns[0].spec.dtype == np.dtype(np.uint8) + assert rebuilt.columns[0].dtype == np.dtype(np.uint8) + + +# --------------------------------------------------------------------------- +# Complex gains nullability, mask-only +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("factory", [blosc2.complex64, blosc2.complex128]) +def test_complex_is_nullable_via_mask(factory): + spec = factory(nullable=True) + assert spec.uses_mask, "complex has no representable sentinel, so nullable implies mask" + + +@pytest.mark.parametrize("factory", [blosc2.complex64, blosc2.complex128]) +def test_complex_rejects_a_sentinel(factory): + with pytest.raises(ValueError, match="cannot use an in-band null sentinel"): + factory(null_value=0j) + + +@pytest.mark.parametrize("factory", [blosc2.complex64, blosc2.complex128]) +def test_complex_non_nullable_is_unchanged(factory): + spec = factory() + assert not spec.nullable + assert spec.to_metadata_dict() == {"kind": spec._kind} + + +# --------------------------------------------------------------------------- +# Serialization and version gating +# --------------------------------------------------------------------------- + + +def _schema(**cols): + fields = [(name, ann, blosc2.field(spec)) for name, (ann, spec) in cols.items()] + return CTable(dataclasses.make_dataclass("Row", fields))._schema + + +def test_sentinel_schema_is_unchanged_on_disk(): + """A sentinel table must serialize exactly as it did before mask storage. + + ``"sentinel"`` is never emitted, so old readers keep working. + """ + d = schema_to_dict(_schema(v=(int, blosc2.int64(null_value=-1)))) + assert d["version"] == 1 + assert d["columns"][0] == {"name": "v", "kind": "int64", "nullable": True, "null_value": -1} + assert "null_storage" not in d["columns"][0] + + +def test_mask_column_records_storage_and_bumps_version(): + d = schema_to_dict(_schema(v=(int, blosc2.int64(null_storage="mask")))) + assert d["version"] == 3 + assert d["columns"][0]["null_storage"] == "mask" + assert "null_value" not in d["columns"][0] + + +def test_version_bump_is_conditional_on_a_mask_column(): + """Only mask-using tables become unreadable by older readers.""" + mixed = _schema( + s=(int, blosc2.int64(null_value=-1)), + plain=(int, blosc2.int64()), + ) + assert schema_to_dict(mixed)["version"] == 1 + + +def test_schema_round_trip_preserves_storage(): + d = schema_to_dict( + _schema( + m=(int, blosc2.int64(null_storage="mask")), + s=(int, blosc2.int64(null_value=-1)), + ) + ) + back = schema_from_dict(d) + assert back.columns_by_name["m"].spec.uses_mask + assert back.columns_by_name["s"].spec.null_value == -1 + assert not back.columns_by_name["s"].spec.uses_mask + assert schema_to_dict(back) == d + + +def test_schema_from_dict_accepts_version_3(): + schema_from_dict({"version": 3, "columns": [{"name": "v", "kind": "int64"}]}) + + +def test_unsupported_version_names_the_version_and_the_way_out(): + with pytest.raises(ValueError, match="Unsupported schema version 9") as exc: + schema_from_dict({"version": 9, "columns": []}) + assert "convert_nulls" in str(exc.value), "the message should say how to get a readable copy" + + +# --------------------------------------------------------------------------- +# NullPolicy +# --------------------------------------------------------------------------- + + +def test_policy_defaults_to_sentinel(): + """Phase 2 ships the capability; the default flips a release later.""" + assert blosc2.NullPolicy().null_storage == "sentinel" + + +def test_policy_rejects_an_unknown_storage(): + with pytest.raises(ValueError, match="null_storage must be 'mask' or 'sentinel'"): + blosc2.NullPolicy(null_storage="bitmap") + + +def test_policy_mask_with_a_type_wide_sentinel_field_is_a_contradiction(): + with pytest.raises(ValueError, match="contradicts the type-wide sentinel field"): + blosc2.NullPolicy(null_storage="mask", string_value="") + + +def test_policy_mask_still_allows_per_column_sentinels(): + """``column_null_values`` forces sentinel storage per column, which is + a refinement of the default rather than a contradiction of it.""" + policy = blosc2.NullPolicy(null_storage="mask", column_null_values={"v": -1}) + assert policy.null_storage == "mask" + + +def test_untouched_nan_float_value_does_not_read_as_set(): + """float_value defaults to NaN, which never equals itself.""" + assert not blosc2.NullPolicy()._sentinel_field_is_set("float_value") + assert blosc2.NullPolicy(float_value=-1.0)._sentinel_field_is_set("float_value") + + +# --------------------------------------------------------------------------- +# Resolution order +# --------------------------------------------------------------------------- + + +def _resolved(spec, annotation=int, name="v", **policy_kw): + Row = dataclasses.make_dataclass("Row", [(name, annotation, blosc2.field(spec))]) + if policy_kw: + with blosc2.null_policy(blosc2.NullPolicy(**policy_kw)): + return CTable(Row)._schema.columns_by_name[name] + return CTable(Row)._schema.columns_by_name[name] + + +def test_explicit_storage_beats_the_policy(): + col = _resolved(blosc2.int64(null_storage="mask"), null_storage="sentinel") + assert col.spec.uses_mask + + +def test_explicit_null_value_implies_sentinel_under_a_mask_policy(): + col = _resolved(blosc2.int64(null_value=-1), null_storage="mask") + assert col.spec.null_value == -1 + assert not col.spec.uses_mask + + +def test_column_null_values_implies_sentinel_under_a_mask_policy(): + col = _resolved(blosc2.int64(nullable=True), null_storage="mask", column_null_values={"v": -1}) + assert col.spec.null_value == -1 + assert not col.spec.uses_mask + + +@pytest.mark.parametrize( + ("field", "value", "spec", "annotation"), + [ + ("string_value", "", blosc2.string(max_length=4, nullable=True), str), + ("bytes_value", b"", blosc2.bytes(max_length=4, nullable=True), bytes), + ("float_value", -1.0, blosc2.float64(nullable=True), float), + ("bool_value", 255, blosc2.bool(nullable=True), bool), + ("timestamp_value", -1, blosc2.timestamp(nullable=True), object), + ("signed_int_strategy", "max", blosc2.int64(nullable=True), int), + ("unsigned_int_strategy", "min", blosc2.uint64(nullable=True), int), + ], +) +def test_type_wide_sentinel_field_implies_sentinel_for_its_kinds(field, value, spec, annotation): + """Existing ``NullPolicy(float_value=...)`` code must keep working once the + default flips to mask, so setting a sentinel field opts those types in. + """ + col = _resolved(spec, annotation, **{field: value}) + assert not col.spec.uses_mask + assert col.spec.null_value is not None + + +def test_policy_mask_applies_to_plain_nullable(): + col = _resolved(blosc2.int64(nullable=True), null_storage="mask") + assert col.spec.uses_mask + assert col.spec.null_value is None + + +def test_mask_skips_string_max_length_widening(): + """The sentinel path widens ``U4`` to fit ``__BLOSC2_NULL__``; mask does not.""" + col = _resolved(blosc2.string(max_length=4, nullable=True), str, null_storage="mask") + assert col.dtype == np.dtype("U4") + + sentinel_col = _resolved(blosc2.string(max_length=4, nullable=True), str) + assert sentinel_col.dtype == np.dtype("U15") + + +def test_mask_skips_the_bool_uint8_flip(): + col = _resolved(blosc2.bool(nullable=True), bool, null_storage="mask") + assert col.dtype == np.dtype(np.bool_) + assert col.spec.dtype == np.dtype(np.bool_) + + +def test_mask_skips_the_ndarray_bool_uint8_flip(): + col = _resolved(blosc2.ndarray((2,), dtype=blosc2.bool(), nullable=True), object, null_storage="mask") + assert col.spec.dtype == np.dtype(np.bool_) + assert col.spec.itemsize == 1 + + +def test_sentinel_default_is_unchanged(): + """The whole point of shipping opt-in first: nothing moves by default.""" + col = _resolved(blosc2.bool(nullable=True), bool) + assert col.dtype == np.dtype(np.uint8) + assert col.spec.null_value == 255 + + +# --------------------------------------------------------------------------- +# fill_value_for +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("spec", "expected"), + [ + (blosc2.int32(), 0), + (blosc2.uint8(), 0), + (blosc2.float32(), None), # NaN, checked separately + (blosc2.complex128(), 0j), + (blosc2.bool(), False), + (blosc2.string(max_length=4), ""), + (blosc2.bytes(max_length=4), b""), + (blosc2.utf8(), ""), + ], +) +def test_fill_value_for(spec, expected): + got = fill_value_for(spec) + if expected is None: + assert np.isnan(got) + else: + assert got == expected + + +def test_fill_value_for_timestamp_decodes_to_nat(): + """int64.min is NaT's bit pattern, so a null timestamp reads back as NaT.""" + fill = fill_value_for(blosc2.timestamp()) + assert fill == np.iinfo(np.int64).min + assert np.isnat(np.array([fill], dtype="datetime64[us]")[0]) + + +def test_fill_value_is_not_recorded_in_the_schema(): + """Recording it would recreate sentinel collisions at the metadata layer.""" + d = schema_to_dict(_schema(v=(float, blosc2.float64(null_storage="mask")))) + assert "fill_value" not in d["columns"][0] + assert "null_value" not in d["columns"][0] + + +# --------------------------------------------------------------------------- +# The declaration survives every persistence route +# --------------------------------------------------------------------------- + +_MASK_ROW_FIELDS = [ + ("m", int, blosc2.field(blosc2.int64(null_storage="mask"))), + ("f", bool, blosc2.field(blosc2.bool(null_storage="mask"))), + ("s", str, blosc2.field(blosc2.string(max_length=4, null_storage="mask"))), + ("c", complex, blosc2.field(blosc2.complex128(nullable=True))), + ("plain", int, blosc2.field(blosc2.int64())), +] +_MASK_PAYLOAD = { + "m": [1, 2], + "f": [True, False], + "s": ["ab", "cd"], + "c": [1 + 2j, 3 + 4j], + "plain": [7, 8], +} +_EXPECTED_STORAGE = {"m": "mask", "f": "mask", "s": "mask", "c": "mask", "plain": None} + + +def _mask_table(): + t = CTable(dataclasses.make_dataclass("MaskRow", _MASK_ROW_FIELDS)) + t.extend(_MASK_PAYLOAD) + return t + + +def _storage_of(table): + return {c.name: c.spec.null_storage for c in table._schema.columns} + + +@pytest.mark.parametrize("ext", [".b2d", ".b2z"]) +def test_storage_survives_save_and_open(tmp_path, ext): + path = tmp_path / f"masked{ext}" + _mask_table().save(str(path)) + reopened = blosc2.open(str(path)) + assert _storage_of(reopened) == _EXPECTED_STORAGE + assert reopened["s"][:].tolist() == ["ab", "cd"] + assert reopened["c"][:].tolist() == [1 + 2j, 3 + 4j] + # A mask column keeps its natural dtype across the round trip. + assert reopened._schema.columns_by_name["f"].dtype == np.dtype(np.bool_) + assert reopened._schema.columns_by_name["s"].dtype == np.dtype("U4") + + +def test_storage_survives_to_cframe(): + restored = blosc2.ctable_from_cframe(_mask_table().to_cframe()) + assert _storage_of(restored) == _EXPECTED_STORAGE + + +def test_storage_survives_copy(): + """Nothing auto-migrates: copy() preserves each column's storage.""" + assert _storage_of(_mask_table().copy()) == _EXPECTED_STORAGE From fbea7c381b4afd5b4a0c4dd090ee54dfb3af22eb Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 08:32:00 +0200 Subject: [PATCH 03/47] Pin the negation corner in both query forms (mask-based-nulls follow-up) Fix the stale _rewrite_null_predicates docstring that still claimed the index path could drop its null post-filtering, contradicting the plan's correction and the retained _exclude_null_positions. Pin the one known string/operator divergence -- the three-valued corner ~((a > 10) & (b == 999)) with a null operand -- as intentional, and record a pre-existing operator-form bug found while pinning it: bare ~(t.a > 10) wrongly returns null rows, because _null_aware_compare collapses null to False at the leaf and ~ inverts it. Documented as a strict xfail and as a dated addendum in the plan's negation blockquote, filed under the deferred Kleene follow-up. Also brings the plan's Phase 0-2 as-built annotations into the tree. Co-Authored-By: Claude Fable 5 --- plans/mask-based-nulls.md | 69 +++++++++++++++++++-- src/blosc2/ctable.py | 9 ++- tests/ctable/test_null_predicate_rewrite.py | 38 ++++++++++++ 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 81f8ec405..faeaacfa5 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,8 +1,10 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS.** Phases 0 and 1 landed 2026-08-08; Phase 1's premise about the index -> path was disproven during implementation and is corrected in place (see §Expression layer). -> Drafted 2026-08-08. +> **Status: IN PROGRESS — Phases 0, 1 and 2 landed 2026-08-08.** Next up is Phase 3 (storage +> sidecar). Two premises were disproven during implementation and are corrected in place, each in +> a blockquote beside the text it corrects: the index path cannot be fixed by a null-aware +> expression (§Expression layer), and the bool dtype-flip cannot move out of `__init__` +> (§Schema layer). Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -134,6 +136,30 @@ until then `valid_array()`/`null_pred()` return `None` and the column reads as n This module is what keeps the work from becoming a 50-site grep, so **it lands first, sentinel-only, with zero behavior change** (Phase 0). +> **As built (2026-08-08).** Three departures from the sketch above, all in the same direction — +> away from snapshotting state that can go stale: +> +> - **Bound to a `Column`, not to `(table, name)`.** Logical reads (`is_null()`, `null_count()`) +> have to honour the column's view — sorted order, row filter — and `Column` already carries +> that. `Column._nulls` builds one on first use and keeps it; a table-level accessor can be +> added in Phase 3 for the physical write paths, which have no `Column`. +> - **`kind` and `fill_value` are properties, not `__slots__` entries.** `_resolve_nullable_specs` +> mutates `spec.null_value` *in place* (`ctable.py:4536`), so a channel that snapshotted at +> construction would report the wrong kind afterwards. Everything reads through to the live +> schema. Pinned by `test_channel_reads_through_to_live_schema`. +> - **`null_mask()` takes no `key` yet.** Dictionary columns answer through `_dictionary_eq`, +> which is whole-column, so a `key` argument would have been silently ignored for them. It +> arrives in Phase 4, when masks make it meaningful. +> +> The `NULL_*` constants and `fill_value_for` ended up **defined in `schema.py`** and re-exported +> here, because this module imports the spec classes from `schema.py` — that is the lower layer. +> Import them from either place. +> +> Also landed here, not in the sketch: `sentinel_mask`, `is_nan_sentinel`, `is_null_value`, +> `kind_of_spec`, `sentinel_guard_expr` and `rewrite_null_predicates` (Phase 1). Unifying the +> "is this sentinel a NaN" test incidentally fixed several sites that spelled it +> `isinstance(nv, float)` and so missed a `float32` NaN sentinel. + ### Schema layer — `src/blosc2/schema.py`, `schema_compiler.py` Every spec re-declares `nullable`/`null_value` today (`_NumericSpec:94`, `timestamp:251`, @@ -153,6 +179,28 @@ Two spec fixes fall out: already flips there (`ctable.py:4543`). Same for `NDArraySpec` bool (`ctable.py:4510-4516`, `4545-4551`). - **`complex64`/`complex128`** (`schema.py:208-238`): gain nullability via the mixin, fill `0j`. +> **Correction (implemented 2026-08-08).** The dtype-flip relocation above is wrong as stated and +> **was not done**. `_resolve_nullable_specs` runs only on the *creation* paths; **opening a stored +> table never calls it** — `open()` rebuilds each spec through `spec_cls(**data)` +> (`schema_compiler.py:447`) and that is the whole of it. Moving the flip out of `__init__` would +> therefore bring every persisted nullable-bool column back as `np.bool_` while its bytes are +> `uint8`, silently misreading `255` as `True`. Verified by instrumenting the resolver across a +> save/open cycle. +> +> What shipped instead splits the responsibility by what each site can know. `__init__` resolves as +> far as metadata alone allows — `nullable and not uses_mask → uint8` — which is exactly the +> information a reopened table carries, so persisted columns come back correct with no resolver +> involved. `_resolve_nullable_specs` then corrects it **in both directions** once the policy has +> spoken, via `_unflip_mask_bool_dtype`: a bare `nullable=True` that resolves to mask gets its +> `uint8` undone. Same split for `NDArraySpec` bool. Regression-pinned by +> `test_stored_uint8_bool_reopens_as_uint8_without_the_resolver`. +> +> Also implemented, beyond what this section specified: **complex is mask-only**. There is no +> complex value safe to reserve, so `complex64(null_value=...)` raises and `nullable=True` resolves +> to mask regardless of the policy default. The spec classes carry a `supports_sentinel` class flag +> for this, which also replaced a `hasattr(spec_cls(), "null_value")` duck-type test in the Arrow +> importer (`ctable.py:7256`) that the mixin would otherwise have made answer True for complex. + **Version gating** — `schema_to_dict` (`schema_compiler.py:487`) computes the version as an explicit feature max: @@ -419,6 +467,19 @@ Phase 1**, landing right after the `NullChannel` refactor and before any mask wo > conservative in one three-valued corner: `not (a > 10 and b == 999)` with a false second term > makes SQL's `NULL AND FALSE` collapse to `FALSE`, so the row should survive, while the guard drops > it. Rows are only ever dropped, never wrongly returned. +> +> **Addendum (2026-08-08): the *operator* form has its own negation leak — a pre-existing bug, not +> a Phase 1 artifact.** `_null_aware_compare` collapses null → False at the comparison leaf and +> returns a plain `LazyExpr`, so `~(t.a > 10)` inverts the collapsed False and **wrongly returns +> null rows** — the failure direction the guarantee above rules out for the string form. Measured: +> `[0, 2]` where SQL and the string form both give `[0]`. It also means the operator form's SQL- +> exact answer in the three-valued corner above is accidental (plain booleans at `~`, not Kleene +> logic). Both behaviors are pinned in `tests/ctable/test_null_predicate_rewrite.py`: +> `test_negation_over_and_corner_is_conservative` (the intentional string/operator divergence) and +> a `strict=True` xfail, `test_operator_form_negation_drops_nulls` (the leak). A real fix needs +> comparison results to carry their null predicate through `~` — a boolean analogue of +> `NullableExpr` with `__invert__` — and folds naturally into decision 8's deferred Kleene +> follow-up rather than Phase 1. ### Groupby @@ -534,7 +595,7 @@ default-created tables require them. |---|---|---|---| | 0 | ✅ **`NullChannel` refactor, sentinel-only.** New `ctable_nulls.py`; route the ~40 `getattr(spec, "null_value")` sites through it across `ctable.py`, `groupby.py`, `ctable_indexing.py`, `schema_validation.py`, `schema_vectorized.py`. Test suite must pass **unmodified**. | M | Low | | 1 | ✅ **Per-leaf null-predicate rewrite** *(storage-independent — pulled forward because it fixes sentinel tables that exist today)*. `_rewrite_null_predicates`, guards emitted inline and only where the sentinel could satisfy the leaf; validity pushed to the negation point. **`_exclude_null_positions` and the indexed-OR bail are retained** — see the correction above; an ordered index never evaluates the predicate, so a null-aware expression cannot fix it. Real payoff: string predicates become null-aware at all. | M | Med | -| 2 | **Schema plumbing.** `_NullableSpecMixin`, `null_storage` kwarg on ~9 specs, conditional version 3, `NullPolicy.null_storage` (**still defaulting to `"sentinel"`**) with sentinel-field inference, `_resolve_nullable_specs` branch, bool/ndarray dtype-flip relocation, `fill_value_for`, complex nullable. | S | Low | +| 2 | ✅ **Schema plumbing.** `_NullableSpecMixin`, `null_storage` kwarg on ~9 specs, conditional version 3, `NullPolicy.null_storage` (**still defaulting to `"sentinel"`**) with sentinel-field inference, `_resolved_null_storage` as the single decision point, `fill_value_for`, complex nullable (mask-only). Dtype-flip relocation **not** done — see the correction above. | S | Low | | 3 | **Storage sidecar.** 5 methods × 4 backends, lazy creation (absent key = all valid); `_grow`/`trim_capacity`/`compact`/`_save_to_storage`/`to_cframe`/`load`; companion-suffix loop in delete/rename. Testable with a hand-built mask, no semantics yet. | M | Low | | 4 | **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`. Mask columns fully usable. | **L** | **High** (`__setitem__`) | | 5 | **Expressions + reductions.** `_raw_null_pred`, `_lazy_nonnull_mask`, `_ndarray_values_for_reduction`, argmin/argmax, `_is_nullable_bool`. Includes the ndarray-propagation gain. | M | Med | diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index f173419bc..922f2108d 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -13146,9 +13146,12 @@ def _rewrite_null_predicates( a lazy expression over the same raw array, so it fuses into the same pass rather than materializing anything. - Making the expression itself null-aware is what lets the index path - drop its own null post-filtering: index and scan now answer from the - same predicate instead of the index correcting for the scan. + This makes the *scan* path correct — including the scan that an + indexed OR over a nullable column bails to. It does not replace the + index path's null post-filtering: an ordered index answers by taking + a range of the sorted column without ever evaluating the predicate, + so ``_exclude_null_positions`` in ``ctable_indexing.py`` stays + load-bearing regardless of how null-aware the expression is. Run this *before* :meth:`_rewrite_nested_expression`, while names in *expr* are still real column names; the injected operand names carry no diff --git a/tests/ctable/test_null_predicate_rewrite.py b/tests/ctable/test_null_predicate_rewrite.py index b466d23fb..20761ba7d 100644 --- a/tests/ctable/test_null_predicate_rewrite.py +++ b/tests/ctable/test_null_predicate_rewrite.py @@ -179,6 +179,44 @@ def test_string_form_agrees_with_operator_form(label, spec, annotation, values, ) +def test_negation_over_and_corner_is_conservative(): + """The one known corner where the two query forms disagree -- pinned. + + ``~((a > 10) & (b == 999))`` with ``a`` null and the second term False: + SQL collapses ``NULL AND FALSE`` to ``FALSE``, so the negation is True and + the row qualifies. The string form guards at the negation point + (``... & (a != -1)``) and drops it. That is by design -- pushing validity + to the negation point is exact for every simpler form, and errs only by + *dropping*: a null row is never wrongly returned (see + plans/mask-based-nulls.md, "Negation caveat, resolved"). + + The operator form returns the SQL answer here, but not by reasoning in + three values: ``_null_aware_compare`` collapses null to False at the leaf, + so ``~`` sees plain booleans -- which is also why it leaks nulls on a bare + negation (the xfail below). If either assertion starts failing, the two + forms moved; make sure they moved *toward* SQL, together. + """ + t = _table(blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1) + + got = sorted(t.where("~((a > 10) & (b == 999))")["b"][:].tolist()) + assert got == [0, 1, 3] # null row dropped; SQL would keep it + + got = sorted(t.where(~((t["a"] > 10) & (t["b"] == 999)))["b"][:].tolist()) + assert got == [0, 1, 2, 3] # SQL-exact + + +@pytest.mark.xfail( + strict=True, + reason="operator-form negation is not null-aware: `a > 10` collapses null to " + "False at the leaf, so ~ turns the null row into a match. The string form " + "gets this right via the negation-point guard. Fixing it needs the " + "comparison result to carry its null predicate through ~.", +) +def test_operator_form_negation_drops_nulls(): + t = _table(blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1) + assert sorted(t.where(~(t["a"] > 10))["b"][:].tolist()) == [0] + + def test_comparing_against_the_sentinel_matches_nothing(): """The sentinel is not a value: SQL has no row whose `a` equals NULL.""" t = _table(blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1) From 182352d3bce45913942f27270b8d0d6e2f9f4670 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 11:23:04 +0200 Subject: [PATCH 04/47] Give mask columns a place to keep their nulls (mask-based-nulls 3) A mask-storage nullable column now has a ``.notnull`` sidecar beside its values: a plain bool NDArray, one byte per physical row, True where the value is not null. Nothing reads or writes one through the public API yet -- that is Phase 4 -- but every path that moves, grows, shrinks or serializes a column now carries it along. Five TableStorage methods across all four backends, plus the companion-suffix loop that generalizes delete_column/rename_column instead of adding a third copy of the utf8 branch. The sidecar is materialized lazily, on the first null actually written, so an absent one is a valid -- and, for a nullable column that has never seen a null, the expected -- state meaning "every row is valid". Null-free nullable columns therefore cost nothing on disk and nothing on the read path. Two things the plan's site list had wrong, both found by testing: * copy()'s in-memory path builds its result through _empty_copy and a per-column gather, never through _save_to_storage, so a copied table would have silently dropped its sidecars. * resize() zero-fills, i.e. *invalid*, so _grow has to write True over the new tail explicitly or every appended row past the old capacity reads as null. Iteration consults storage rather than the open handles: a freshly reopened table has opened no sidecar, and skipping an unopened one would leave it out of step with its column. Co-Authored-By: Claude Opus 5 --- plans/mask-based-nulls.md | 42 ++- src/blosc2/ctable.py | 252 +++++++++++++- src/blosc2/ctable_storage.py | 196 ++++++++++- tests/ctable/test_null_persistence.py | 461 ++++++++++++++++++++++++++ 4 files changed, 928 insertions(+), 23 deletions(-) create mode 100644 tests/ctable/test_null_persistence.py diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index faeaacfa5..f45e6d4b6 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,10 +1,10 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS — Phases 0, 1 and 2 landed 2026-08-08.** Next up is Phase 3 (storage -> sidecar). Two premises were disproven during implementation and are corrected in place, each in -> a blockquote beside the text it corrects: the index path cannot be fixed by a null-aware -> expression (§Expression layer), and the bool dtype-flip cannot move out of `__init__` -> (§Schema layer). Drafted 2026-08-08. +> **Status: IN PROGRESS — Phases 0–3 landed 2026-08-08.** Next up is Phase 4 (read/write + null +> API), the large, high-risk one. Two premises were disproven during implementation and are +> corrected in place, each in a blockquote beside the text it corrects: the index path cannot be +> fixed by a null-aware expression (§Expression layer), and the bool dtype-flip cannot move out of +> `__init__` (§Schema layer). Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -278,6 +278,36 @@ and **`CTable.load` (`:6099`), a second parallel open path that is easy to miss* operate only on masks that exist — an absent sidecar needs no growing, trimming, or copying, which is most of the lazy-materialization payoff. +> **As built (2026-08-08).** All five storage methods, four backends, both companion-suffix loops, +> and every capacity/persistence site above. Six notes, the first two of which are corrections: +> +> - **The site list was incomplete.** `copy()`'s in-memory path (`ctable.py:12563`) builds its +> result through `_empty_copy` + a per-column gather and never touches `_save_to_storage`, so a +> copied table would have silently dropped its sidecars. It gathers them by the same `live_pos` +> now. `to_b2z`/`to_b2d` need nothing: their physical-pack fast paths zip the TreeStore leaves +> as-is, and `.notnull` is one of them. +> - **`resize()` zero-fills, i.e. *invalid*.** `_grow` must write `True` over the new tail +> explicitly, or every appended row past the old capacity reads as null. Same for the freed tail +> in `compact`. This is the one place where "absent means all-valid" and "present means read the +> bytes" have to be reconciled by hand. Pinned by `test_grow_extends_the_sidecar_as_all_valid`. +> - **Iteration must consult storage, not just the cache.** `_existing_null_masks()` asks the +> storage backend per mask-storage column rather than iterating what happens to be open; +> `_grow`/`trim_capacity` on a *freshly reopened* table have opened nothing yet, and skipping an +> unopened sidecar would leave it out of step with its column. Pinned by +> `test_capacity_paths_find_an_unopened_sidecar`. +> - **Chunk pinning has a documented fallback.** `_null_mask_grid` pins to the value column's +> `chunks[0]`/`blocks[0]`, dictionary columns to their `codes`, and anything whose payload is not +> a plain row-indexed NDArray — utf8, whose offsets array carries `n + 1` entries — to the +> table-wide `_valid_rows` grid, which is the same shared grid the fixed-width columns use. +> `_save_to_storage` records the grid each column *actually landed on* (`dest_grids`) rather than +> re-deriving it, because `chunks_override` and the reblock fast path can both change it. +> - **`_null_masks` is a property that resolves through `base`**, so the six view-construction +> sites that build a `CTable` via `__new__` needed no edits: a view shares its base's sidecars +> exactly as it already shares its `_cols` NDArrays. +> - **Rename drops the cached handle on disk, carries it in memory.** `storage.rename_column` +> re-keys the sidecar, so a cached handle points at a key that no longer exists; in-memory +> storage re-keys nothing, so there the handle *is* the sidecar and must be carried. + ### Read / write paths Today fixed-width scalar columns cannot accept `None` at all — `_coerce_row_to_storage`'s @@ -596,7 +626,7 @@ default-created tables require them. | 0 | ✅ **`NullChannel` refactor, sentinel-only.** New `ctable_nulls.py`; route the ~40 `getattr(spec, "null_value")` sites through it across `ctable.py`, `groupby.py`, `ctable_indexing.py`, `schema_validation.py`, `schema_vectorized.py`. Test suite must pass **unmodified**. | M | Low | | 1 | ✅ **Per-leaf null-predicate rewrite** *(storage-independent — pulled forward because it fixes sentinel tables that exist today)*. `_rewrite_null_predicates`, guards emitted inline and only where the sentinel could satisfy the leaf; validity pushed to the negation point. **`_exclude_null_positions` and the indexed-OR bail are retained** — see the correction above; an ordered index never evaluates the predicate, so a null-aware expression cannot fix it. Real payoff: string predicates become null-aware at all. | M | Med | | 2 | ✅ **Schema plumbing.** `_NullableSpecMixin`, `null_storage` kwarg on ~9 specs, conditional version 3, `NullPolicy.null_storage` (**still defaulting to `"sentinel"`**) with sentinel-field inference, `_resolved_null_storage` as the single decision point, `fill_value_for`, complex nullable (mask-only). Dtype-flip relocation **not** done — see the correction above. | S | Low | -| 3 | **Storage sidecar.** 5 methods × 4 backends, lazy creation (absent key = all valid); `_grow`/`trim_capacity`/`compact`/`_save_to_storage`/`to_cframe`/`load`; companion-suffix loop in delete/rename. Testable with a hand-built mask, no semantics yet. | M | Low | +| 3 | ✅ **Storage sidecar.** 5 methods × 4 backends, lazy creation (absent key = all valid); `_grow`/`trim_capacity`/`compact`/`_save_to_storage`/`to_cframe`/`load`, **plus `copy()`'s in-memory path**, which the section above had missed; companion-suffix loop in delete/rename. `tests/ctable/test_null_persistence.py` (38 tests) drives it with a hand-built mask. | M | Low | | 4 | **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`. Mask columns fully usable. | **L** | **High** (`__setitem__`) | | 5 | **Expressions + reductions.** `_raw_null_pred`, `_lazy_nonnull_mask`, `_ndarray_values_for_reduction`, argmin/argmax, `_is_nullable_bool`. Includes the ndarray-propagation gain. | M | Med | | 6 | **Arrow/Parquet.** Import + export for all V1 kinds, `packbits`/`unpackbits` LSB-first, `arrow_slice(validity=)`, delete the "no sentinel available" import error. Ships **opt-in** (`null_storage="mask"`); the default stays `"sentinel"`. | M | Med | diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 922f2108d..ae1e2037f 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -3773,6 +3773,63 @@ def __delitem__(self, name: str) -> None: dict.__delitem__(self, name) +class _NullMaskCache: + """Open-on-demand cache of the per-column ``.notnull`` validity sidecars. + + Mirrors :class:`_LazyColumnDict`: a wide table should not pay one + ``storage.open_null_mask()`` per nullable column just to be opened. + + A miss is a normal state, not an error. A mask-storage column that has + never held a null has no sidecar at all -- an absent one means *every row + is valid* -- so :meth:`get` answers ``None`` there and callers read the + column as never-null. Negative answers are remembered too, so a null-free + column costs one membership test for the life of the table rather than one + per access. + """ + + __slots__ = ("_absent", "_masks", "_storage") + + def __init__(self, storage: TableStorage) -> None: + self._storage = storage + self._masks: dict[str, blosc2.NDArray] = {} + self._absent: set[str] = set() + + def get(self, name: str) -> blosc2.NDArray | None: + """The sidecar for column *name*, or ``None`` when it has none.""" + mask = self._masks.get(name) + if mask is not None or name in self._absent: + return mask + try: + present = self._storage.has_null_mask(name) + except (NotImplementedError, RuntimeError): + present = False + if not present: + self._absent.add(name) + return None + mask = self._storage.open_null_mask(name) + self._masks[name] = mask + return mask + + def set(self, name: str, mask: blosc2.NDArray) -> None: + self._masks[name] = mask + self._absent.discard(name) + + def pop(self, name: str) -> None: + self._masks.pop(name, None) + self._absent.discard(name) + + def rename(self, old: str, new: str) -> None: + mask = self._masks.pop(old, None) + self._absent.discard(old) + self._absent.discard(new) + if mask is not None: + self._masks[new] = mask + + def materialized(self) -> dict[str, blosc2.NDArray]: + """The sidecars opened so far, without opening any more.""" + return dict(self._masks) + + class _ChunkAlignedWriter: """Buffer writes to a fixed-size NDArray and flush them chunk-aligned. @@ -4943,6 +5000,109 @@ def _open_column_from_storage(self, storage: TableStorage, name: str): return storage.open_dictionary_column(name, cc.spec) return storage.open_column(name) + # ------------------------------------------------------------------ + # Per-column validity sidecars + # ------------------------------------------------------------------ + + @property + def _null_masks(self) -> _NullMaskCache: + """Open-on-demand cache of this table's ``.notnull`` validity sidecars. + + A view shares its base table's cache: sidecars are physical, + row-indexed arrays, exactly like the ``_cols`` NDArrays a view already + shares with its base. + """ + base = self.base + if base is not None: + return base._null_masks + cache = self.__dict__.get("_null_mask_cache") + if cache is None: + cache = _NullMaskCache(self._storage) + self.__dict__["_null_mask_cache"] = cache + return cache + + @property + def _null_mask_names(self) -> list[str]: + """Columns whose schema says their nulls live in a sidecar. + + Being listed here says nothing about whether a sidecar *exists*: one + is written only once a null actually is, so most entries answer + ``None`` from :meth:`_null_mask`. + """ + return [c.name for c in self._schema.columns if getattr(c.spec, "uses_mask", False)] + + def _null_mask(self, name: str) -> blosc2.NDArray | None: + """Column *name*'s validity sidecar, or ``None`` when it has none. + + ``None`` means *every row is valid*, not "unknown" — see + :class:`_NullMaskCache`. + """ + return self._null_masks.get(name) + + def _null_mask_grid(self, name: str, capacity: int) -> tuple[tuple[int], tuple[int]]: + """Chunk/block grid for column *name*'s validity sidecar. + + **Pinned to the value column's row grid**, not to whatever + ``compute_chunks_blocks`` would pick for a one-byte bool dtype. A + sidecar on its own grid would make every paired read — the + chunk-aligned writers, ``_nonnull_chunks``, index-segment alignment — + re-align mask against values on each pass. + + Falls back to the table-wide ``_valid_rows`` grid (which is the same + shared grid the fixed-width columns use) for columns whose payload is + not a plain row-indexed NDArray, such as utf8, whose offsets array + carries ``n + 1`` entries. + """ + cc = self._schema.columns_by_name[name] + arr = self._cols[name] + if self._is_dictionary_column(cc): + arr = arr.codes + elif not isinstance(arr, blosc2.NDArray): + arr = self._valid_rows + chunks = (max(1, min(int(arr.chunks[0]), capacity)),) + blocks = (max(1, min(int(arr.blocks[0]), chunks[0])),) + return chunks, blocks + + def _ensure_null_mask(self, name: str) -> blosc2.NDArray: + """Return column *name*'s validity sidecar, materializing it if absent. + + This is the one place that turns a null-free mask column into one with + bytes on disk: creation is deferred to the first null actually + written, since until then the column's absent sidecar already says + exactly the right thing. + """ + if self.base is not None: + return self.base._ensure_null_mask(name) + mask = self._null_masks.get(name) + if mask is not None: + return mask + capacity = len(self._valid_rows) + chunks, blocks = self._null_mask_grid(name, capacity) + mask = self._storage.create_null_mask(name, shape=(capacity,), chunks=chunks, blocks=blocks) + self._null_masks.set(name, mask) + return mask + + def _existing_null_masks(self) -> dict[str, blosc2.NDArray]: + """Every validity sidecar this table actually has, opening as needed. + + Unlike ``_null_masks.materialized()`` this consults storage for each + mask-storage column, so it is the iterator the capacity and + persistence paths must use: they cannot miss a sidecar merely because + nothing has read that column yet. + """ + found = {} + for name in self._null_mask_names: + mask = self._null_masks.get(name) + if mask is not None: + found[name] = mask + return found + + def _drop_null_mask(self, name: str) -> None: + """Forget and (for persistent tables) delete column *name*'s sidecar.""" + self._null_masks.pop(name) + with contextlib.suppress(NotImplementedError, RuntimeError, KeyError): + self._storage.delete_null_mask(name) + def _resolve_last_pos(self) -> int: """Return the physical index of the next write slot. @@ -5005,6 +5165,10 @@ def trim_capacity(self) -> None: col_arr.resize((target,)) continue col_arr.resize(self._column_physical_shape(cc, target)) + # Only sidecars that exist need trimming; an absent one has no bytes to + # reclaim, which is most of the payoff of materializing them lazily. + for mask in self._existing_null_masks().values(): + mask.resize((target,)) self._valid_rows.resize((target,)) self._last_pos = target @@ -5021,6 +5185,13 @@ def _grow(self) -> None: col_arr.resize((new_capacity,)) continue col_arr.resize(self._column_physical_shape(cc, new_capacity)) + # Grow the validity sidecars alongside their columns. resize() zero-fills, + # i.e. *invalid*, so the new tail must be marked valid explicitly to keep + # the "no null written yet" reading of those rows. + for mask in self._existing_null_masks().values(): + old_capacity = mask.shape[0] + mask.resize((new_capacity,)) + mask[old_capacity:new_capacity] = True self._valid_rows.resize((new_capacity,)) # ------------------------------------------------------------------ @@ -5894,6 +6065,7 @@ def to_cframe(self) -> bytes: from blosc2.ctable_storage import ( _DICT_SUFFIX, + _NOTNULL_SUFFIX, _UTF8_DATA_SUFFIX, _column_name_to_relpath, ) @@ -5931,6 +6103,11 @@ def to_cframe(self) -> bytes: # Scalar NDArray or ListArray — both serialize via to_cframe(). estore[key] = arr + # Validity sidecars travel beside their columns; a column without one + # simply contributes no entry, which reconstructs as all-valid. + for name, mask in src._existing_null_masks().items(): + estore[f"/_cols/{_column_name_to_relpath(name)}{_NOTNULL_SUFFIX}"] = mask + return estore.to_cframe() def _save_to_storage( # noqa: C901 @@ -5965,14 +6142,21 @@ def _save_to_storage( # noqa: C901 ) # --- valid_rows (all True, compacted) --- + valid_chunks = shared_chunks if shared_chunks is not None else default_chunks + valid_blocks = shared_blocks if shared_blocks is not None else default_blocks disk_valid = storage.create_valid_rows( shape=(capacity,), - chunks=shared_chunks if shared_chunks is not None else default_chunks, - blocks=shared_blocks if shared_blocks is not None else default_blocks, + chunks=valid_chunks, + blocks=valid_blocks, ) if n_live > 0: disk_valid[:n_live] = True + # Row grid each fixed-width column actually landed on, so its validity + # sidecar below can be pinned to that same grid rather than re-derived + # (chunk overrides and the reblock path can both change it). + dest_grids: dict[str, tuple[tuple[int], tuple[int]]] = {} + # --- columns --- for col in self._schema.columns: name = col.name @@ -6060,6 +6244,7 @@ def _save_to_storage( # noqa: C901 copy_kwargs["cparams"] = cparams_override new_arr = src_arr.copy(**copy_kwargs) storage.install_column(name, new_arr) + dest_grids[name] = ((int(new_arr.chunks[0]),), (int(new_arr.blocks[0]),)) else: eff_chunks = chunks_override if chunks_override is not None else col_storage["chunks"] if chunks_override is not None and blocks_override is None: @@ -6079,10 +6264,23 @@ def _save_to_storage( # noqa: C901 cparams=cparams_override if cparams_override is not None else col_storage.get("cparams"), dparams=col_storage.get("dparams"), ) + dest_grids[name] = ((int(disk_col.chunks[0]),), (int(disk_col.blocks[0]),)) if n_live > 0: # Slice is ~30x faster than fancy-index for sequential no-deletion access. disk_col[:n_live] = src_arr[:n_live] if no_deletions else src_arr[live_pos] + # --- validity sidecars (compacted alongside their columns) --- + # Written after the columns so each can be pinned to the grid its column + # actually landed on. A column whose sidecar is absent stays absent: + # no nulls, no bytes, and nothing to copy. + for name, mask in self._existing_null_masks().items(): + mask_chunks, mask_blocks = dest_grids.get(name, (valid_chunks, valid_blocks)) + disk_mask = storage.create_null_mask( + name, shape=(capacity,), chunks=mask_chunks, blocks=mask_blocks + ) + if n_live > 0: + disk_mask[:n_live] = mask[:n_live] if no_deletions else mask[live_pos] + storage.save_schema(self._schema_dict_with_computed()) def save(self, urlpath: str, *, overwrite: bool = False) -> None: @@ -6305,6 +6503,28 @@ def load(cls, urlpath: str) -> CTable: # noqa: C901 mem_col[:phys_size] = disk_cols[name][:] mem_cols[name] = mem_col + # Validity sidecars, pinned to the grid their in-memory column got. + # Columns with no stored sidecar are left without one — that already + # means all-valid, so there is nothing to load. + mem_masks: dict[str, blosc2.NDArray] = {} + for col in schema.columns: + name = col.name + if not getattr(col.spec, "uses_mask", False) or not file_storage.has_null_mask(name): + continue + src_mask = file_storage.open_null_mask(name) + payload = mem_cols[name] + grid_src = payload if isinstance(payload, blosc2.NDArray) else mem_valid + mask_chunks = max(1, min(int(grid_src.chunks[0]), capacity)) + mem_mask = mem_storage.create_null_mask( + name, + shape=(capacity,), + chunks=(mask_chunks,), + blocks=(max(1, min(int(grid_src.blocks[0]), mask_chunks)),), + ) + if phys_size > 0: + mem_mask[:phys_size] = src_mask[:phys_size] + mem_masks[name] = mem_mask + file_storage.close() obj = cls.__new__(cls) @@ -6323,6 +6543,8 @@ def load(cls, urlpath: str) -> CTable: # noqa: C901 obj._summary_indexes_built = schema_dict.get("summary_indexes_built", False) obj.base = None obj._valid_rows = mem_valid + for mask_name, mask in mem_masks.items(): + obj._null_masks.set(mask_name, mask) obj._n_rows = n_live obj._last_pos = None # resolve lazily on first write obj._computed_cols = {} @@ -9404,7 +9626,10 @@ def drop_column(self, name: str) -> None: self._invalidate_index_catalog_cache() if isinstance(self._storage, FileTableStorage): + # delete_column already removes the column's companion keys, the + # validity sidecar among them; drop the cached handle to match. self._storage.delete_column(name) + self._null_masks.pop(name) self._materialized_cols.pop(name, None) del self._cols[name] @@ -9484,8 +9709,15 @@ def rename_column(self, old: str, new: str) -> None: if isinstance(self._storage, FileTableStorage): self._cols[new] = self._rename_stored_column(old, new) + # storage.rename_column re-keyed the validity sidecar too, leaving + # any cached handle pointing at a key that no longer exists. Drop + # it and let the cache reopen under the new name. + self._null_masks.pop(old) + self._null_masks.pop(new) else: self._cols[new] = self._cols[old] + # Nothing was re-keyed on disk (there is no disk): carry the handle. + self._null_masks.rename(old, new) del self._cols[old] idx = self.col_names.index(old) @@ -11554,6 +11786,13 @@ def compact(self): start += block_size end = min(end + block_size, self._n_rows) + # Shuffle the validity sidecars by the same gather, so mask and values + # stay row-aligned. The freed tail becomes all-valid: those slots hold + # no live row, and "valid" is what an absent sidecar would say of them. + for mask in self._existing_null_masks().values(): + mask[: self._n_rows] = mask[real_poss[: self._n_rows]] + mask[self._n_rows :] = True + self._valid_rows[: self._n_rows] = True self._valid_rows[self._n_rows :] = False self._last_pos = self._n_rows @@ -12323,6 +12562,15 @@ def copy( # noqa: C901 arr[:n_live] if is_dense else (arr[live_pos] if compact else arr[:n]) ) + # Validity sidecars follow their columns through the same gather. A + # column with no sidecar gets none: nothing in it is null, and copying + # must not invent storage the source never needed (decision 9). + if n > 0: + for col_name, mask in self._existing_null_masks().items(): + result._ensure_null_mask(col_name)[:n] = ( + mask[:n_live] if is_dense else (mask[live_pos] if compact else mask[:n]) + ) + if compact: result._valid_rows[:n] = True result._n_rows = n diff --git a/src/blosc2/ctable_storage.py b/src/blosc2/ctable_storage.py index 0d01b4d44..c8813f1e0 100644 --- a/src/blosc2/ctable_storage.py +++ b/src/blosc2/ctable_storage.py @@ -141,6 +141,46 @@ def create_valid_rows( def open_valid_rows(self) -> blosc2.NDArray: raise NotImplementedError + # -- Per-column validity sidecars ---------------------------------------- + # + # A mask-storage nullable column keeps its nullity in a bool NDArray beside + # the values (``True`` = not null). The sidecar is materialized on the + # first null actually written, never at column creation, so + # ``has_null_mask(name) is False`` is the common case and means *all rows + # valid* -- a null-free nullable column costs nothing on disk and nothing + # on the read path. + + def create_null_mask( + self, + name: str, + *, + shape: tuple[int, ...], + chunks: tuple[int, ...], + blocks: tuple[int, ...], + ) -> blosc2.NDArray: + """Create column *name*'s validity sidecar, initialized all-valid. + + Filled with ``True`` rather than zeros so that a freshly materialized + sidecar says exactly what its absence said. + """ + raise NotImplementedError + + def install_null_mask(self, name: str, ndarray: blosc2.NDArray) -> blosc2.NDArray: + """Store a pre-built bool NDArray as column *name*'s validity sidecar.""" + raise NotImplementedError + + def open_null_mask(self, name: str) -> blosc2.NDArray: + """Open column *name*'s validity sidecar. Call only if it exists.""" + raise NotImplementedError + + def has_null_mask(self, name: str) -> bool: + """Whether column *name* has a stored validity sidecar.""" + raise NotImplementedError + + def delete_null_mask(self, name: str) -> None: + """Remove column *name*'s validity sidecar if it has one.""" + raise NotImplementedError + def save_schema(self, schema_dict: dict[str, Any]) -> None: raise NotImplementedError @@ -285,6 +325,23 @@ def create_valid_rows(self, *, shape, chunks, blocks): def open_valid_rows(self): raise RuntimeError("In-memory tables have no on-disk representation to open.") + def create_null_mask(self, name, *, shape, chunks, blocks): + return blosc2.full(shape, True, dtype=np.bool_, chunks=chunks, blocks=blocks) + + def install_null_mask(self, name, ndarray: blosc2.NDArray) -> blosc2.NDArray: + return ndarray + + def open_null_mask(self, name): + raise RuntimeError("In-memory tables have no on-disk representation to open.") + + def has_null_mask(self, name) -> bool: + # In-memory sidecars are held by the CTable, not by this storage, so + # there is never one to discover here — mirroring open_column. + return False + + def delete_null_mask(self, name) -> None: + pass # nothing persisted to remove + def save_schema(self, schema_dict): pass # nothing to persist @@ -409,6 +466,23 @@ def _column_name_to_relpath(name: str) -> str: # dots are percent-encoded, so no column name maps to a key containing ".". _UTF8_DATA_SUFFIX = ".utf8" +# Key suffix for the validity sidecar of a mask-storage nullable column: a +# plain bool NDArray, one byte per physical row, ``True`` where the value is +# *not* null (Arrow's polarity, which the name states). Deliberately not +# ``.valid``: ``/_valid_rows`` already exists and means row liveness, not +# per-column nullity, and two bool arrays with near-identical names would +# invite conflating them. Collision-free by the same argument as +# ``_UTF8_DATA_SUFFIX`` above. +# +# The sidecar is created lazily, on the first write that actually contains a +# null, so its *absence* is a valid -- and, for a nullable column that has +# never seen a null, the expected -- state meaning "every row is valid". +_NOTNULL_SUFFIX = ".notnull" + +# Per-column companion keys that live beside a column's own key and must be +# carried along by any operation that moves or removes the column. +_COMPANION_KEY_SUFFIXES = (_UTF8_DATA_SUFFIX, _NOTNULL_SUFFIX) + class EmbedStoreTableStorage(TableStorage): """Read-only :class:`CTable` storage backed by an in-memory :class:`blosc2.EmbedStore`. @@ -505,6 +579,14 @@ def open_dictionary_column(self, name: str, spec) -> DictionaryColumn: def open_valid_rows(self) -> blosc2.NDArray: return self._estore[_VALID_ROWS_KEY] + # -- per-column validity sidecars -------------------------------------- + + def open_null_mask(self, name: str) -> blosc2.NDArray: + return self._estore[self._col_key(name) + _NOTNULL_SUFFIX] + + def has_null_mask(self, name: str) -> bool: + return (self._col_key(name) + _NOTNULL_SUFFIX) in self._estore + # -- status ----------------------------------------------------------- def table_exists(self) -> bool: @@ -531,6 +613,9 @@ def _not_supported(self, *args, **kwargs): create_varlen_scalar_column = _not_supported create_dictionary_column = _not_supported create_valid_rows = _not_supported + create_null_mask = _not_supported + install_null_mask = _not_supported + delete_null_mask = _not_supported save_schema = _not_supported save_vlmeta = _not_supported delete_column = _not_supported @@ -820,6 +905,32 @@ def create_valid_rows(self, *, shape, chunks, blocks): def open_valid_rows(self) -> blosc2.NDArray: return self._open_store()[_VALID_ROWS_KEY] + def _notnull_key(self, name: str) -> str: + return self._col_key(name) + _NOTNULL_SUFFIX + + def create_null_mask(self, name, *, shape, chunks, blocks) -> blosc2.NDArray: + mask = blosc2.full(shape, True, dtype=np.bool_, chunks=chunks, blocks=blocks) + store = self._open_store() + store[self._notnull_key(name)] = mask + return store[self._notnull_key(name)] + + def install_null_mask(self, name, ndarray: blosc2.NDArray) -> blosc2.NDArray: + store = self._open_store() + store[self._notnull_key(name)] = ndarray + return store[self._notnull_key(name)] + + def open_null_mask(self, name: str) -> blosc2.NDArray: + return self._open_store()[self._notnull_key(name)] + + def has_null_mask(self, name: str) -> bool: + return self._notnull_key(name) in self._open_store() + + def delete_null_mask(self, name: str) -> None: + store = self._open_store() + key = self._notnull_key(name) + if key in store: + del store[key] + def save_schema(self, schema_dict: dict[str, Any]) -> None: """Write *schema_dict* (plus kind/version markers) to ``/_meta``.""" meta = blosc2.SChunk() @@ -899,12 +1010,13 @@ def column_names_from_schema(self) -> list[str]: return [c["name"] for c in d["columns"]] def delete_column(self, name: str) -> None: + store = self._open_store() key = self._col_key(name) - if key in self._open_store(): - del self._open_store()[key] - data_key = key + _UTF8_DATA_SUFFIX - if data_key in self._open_store(): - del self._open_store()[data_key] + if key in store: + del store[key] + for suffix in _COMPANION_KEY_SUFFIXES: + if key + suffix in store: + del store[key + suffix] return list_path = self._list_col_path(name) if os.path.exists(list_path): @@ -919,10 +1031,10 @@ def rename_column(self, old: str, new: str): if old_key in store: store[new_key] = store[old_key] del store[old_key] - old_data_key = old_key + _UTF8_DATA_SUFFIX - if old_data_key in store: - store[new_key + _UTF8_DATA_SUFFIX] = store[old_data_key] - del store[old_data_key] + for suffix in _COMPANION_KEY_SUFFIXES: + if old_key + suffix in store: + store[new_key + suffix] = store[old_key + suffix] + del store[old_key + suffix] return store[new_key] old_path = self._list_col_path(old) new_path = self._list_col_path(new) @@ -1426,6 +1538,58 @@ def create_valid_rows( def open_valid_rows(self) -> blosc2.NDArray: return self._open_leaf("/_valid_rows") + # -- per-column validity sidecars -------------------------------------- + + def _notnull_logical_key(self, name: str) -> str: + return self._col_logical_key(name) + _NOTNULL_SUFFIX + + def _register_leaf(self, logical_key: str, dest_path: str) -> None: + """Point the outer store's map_tree at an external leaf file.""" + rel_path = os.path.relpath(dest_path, self._working_dir()).replace(os.sep, "/") + self._store.map_tree[self._table_key(logical_key)] = rel_path + self._store._modified = True + + def create_null_mask(self, name, *, shape, chunks, blocks) -> blosc2.NDArray: + logical_key = self._notnull_logical_key(name) + dest_path = self._dest_path(logical_key, ".b2nd") + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + mask = blosc2.full( + shape, + True, + dtype=np.bool_, + chunks=chunks, + blocks=blocks, + urlpath=dest_path, + mode="w", + ) + self._register_leaf(logical_key, dest_path) + return mask + + def install_null_mask(self, name, ndarray: blosc2.NDArray) -> blosc2.NDArray: + logical_key = self._notnull_logical_key(name) + dest_path = self._dest_path(logical_key, ".b2nd") + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + saved = ndarray.copy(urlpath=dest_path) + self._register_leaf(logical_key, dest_path) + return saved + + def open_null_mask(self, name: str) -> blosc2.NDArray: + return self._open_leaf(self._notnull_logical_key(name)) + + def has_null_mask(self, name: str) -> bool: + full_key = self._table_key(self._notnull_logical_key(name)) + return full_key in self._store.map_tree or full_key in self._store._estore + + def delete_null_mask(self, name: str) -> None: + full_key = self._table_key(self._notnull_logical_key(name)) + if full_key not in self._store.map_tree: + return + filepath = self._store.map_tree.pop(full_key) + full_path = os.path.join(self._working_dir(), filepath) + if os.path.exists(full_path): + os.remove(full_path) + self._store._modified = True + # ------------------------------------------------------------------ # TableStorage interface — schema and manifest # ------------------------------------------------------------------ @@ -1497,9 +1661,10 @@ def delete_column(self, name: str) -> None: full_key = self._table_key(self._col_logical_key(name)) if full_key in self._store.map_tree: keys = [full_key] - data_key = self._table_key(self._col_logical_key(name) + _UTF8_DATA_SUFFIX) - if data_key in self._store.map_tree: - keys.append(data_key) + for suffix in _COMPANION_KEY_SUFFIXES: + companion = self._table_key(self._col_logical_key(name) + suffix) + if companion in self._store.map_tree: + keys.append(companion) for key in keys: filepath = self._store.map_tree.pop(key) full_path = os.path.join(self._working_dir(), filepath) @@ -1516,9 +1681,10 @@ def rename_column(self, old: str, new: str) -> blosc2.NDArray: old_key = self._table_key(self._col_logical_key(old)) if old_key in self._store.map_tree: moves = [(old_key, self._col_logical_key(new))] - old_data_key = self._table_key(self._col_logical_key(old) + _UTF8_DATA_SUFFIX) - if old_data_key in self._store.map_tree: - moves.append((old_data_key, self._col_logical_key(new) + _UTF8_DATA_SUFFIX)) + for suffix in _COMPANION_KEY_SUFFIXES: + companion = self._table_key(self._col_logical_key(old) + suffix) + if companion in self._store.map_tree: + moves.append((companion, self._col_logical_key(new) + suffix)) new_dest = None for src_key, dst_logical in moves: dst_dest = self._dest_path(dst_logical, ".b2nd") diff --git a/tests/ctable/test_null_persistence.py b/tests/ctable/test_null_persistence.py new file mode 100644 index 000000000..82985a432 --- /dev/null +++ b/tests/ctable/test_null_persistence.py @@ -0,0 +1,461 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""The ``.notnull`` validity sidecar, through every persistence path (Phase 3). + +A mask-storage nullable column keeps its nullity in a bool NDArray beside the +values. Nothing reads or writes one through the public API yet -- ``extend``, +``is_null`` and friends arrive in Phase 4 -- so these tests build the sidecar +by hand via ``_ensure_null_mask`` and check that every path that moves, grows, +shrinks or serializes a column carries it along unchanged. + +The load-bearing invariants here: + +* **An absent sidecar means all-valid.** A nullable column that has never + been given a null has no ``.notnull`` key at all, and must not acquire one + just by being saved, copied or reopened. +* **Chunk pinning.** The sidecar shares its column's row grid, so mask and + values never need re-aligning on a paired read. +""" + +from __future__ import annotations + +import dataclasses +import os + +import numpy as np +import pytest + +import blosc2 +from blosc2.ctable_storage import _NOTNULL_SUFFIX, FileTableStorage + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + +NULL_POSITIONS = (3, 7, 41) + + +def make_row_type(): + return dataclasses.make_dataclass( + "MaskRow", + [ + ("a", int, blosc2.field(blosc2.int64(null_storage="mask"))), + ("b", float, blosc2.field(blosc2.float64())), + ("c", str, blosc2.field(blosc2.string(max_length=8, null_storage="mask"))), + ], + ) + + +def make_table(n_rows=50, capacity=100, *, with_nulls=True, urlpath=None): + """A table with two mask columns; only ``a`` is given nulls.""" + kwargs = {"expected_size": capacity} + if urlpath is not None: + kwargs |= {"urlpath": urlpath, "mode": "w"} + t = blosc2.CTable(make_row_type(), **kwargs) + t.extend([(i, float(i), f"s{i}") for i in range(n_rows)]) + if with_nulls: + mask = t._ensure_null_mask("a") + for pos in NULL_POSITIONS: + if pos < n_rows: + mask[pos] = False + return t + + +def null_positions(table, name="a", n=None): + """Physical positions the sidecar marks null, or ``None`` if there is none.""" + mask = table._null_mask(name) + if mask is None: + return None + n = table._n_rows if n is None else n + return np.flatnonzero(~mask[:n]).tolist() + + +def sidecar_key_exists(table, name): + storage = table._storage + return storage.has_null_mask(name) + + +# --------------------------------------------------------------------------- +# Decision 9: an absent sidecar means all-valid +# --------------------------------------------------------------------------- + + +def test_a_fresh_mask_column_has_no_sidecar(): + t = make_table(with_nulls=False) + assert t._null_mask_names == ["a", "c"] + assert t._null_mask("a") is None + assert t._null_mask("c") is None + assert t._existing_null_masks() == {} + + +def test_non_nullable_columns_are_not_listed(): + t = make_table(with_nulls=False) + assert "b" not in t._null_mask_names + assert t._null_mask("b") is None + + +def test_sentinel_columns_are_not_listed(): + Row = dataclasses.make_dataclass("SentinelRow", [("v", int, blosc2.field(blosc2.int64(null_value=-1)))]) + t = blosc2.CTable(Row, expected_size=8) + assert t._null_mask_names == [] + + +def test_materializing_the_sidecar_starts_all_valid(): + t = make_table(with_nulls=False) + mask = t._ensure_null_mask("a") + assert mask.shape == (len(t._valid_rows),) + assert mask.dtype == np.dtype(np.bool_) + # An explicit all-True fill, not zeros: a fresh sidecar must say exactly + # what its absence said. + assert bool(np.asarray(mask[:]).all()) + + +def test_ensure_null_mask_is_idempotent(): + t = make_table(with_nulls=False) + first = t._ensure_null_mask("a") + first[5] = False + second = t._ensure_null_mask("a") + assert second is first + assert null_positions(t) == [5] + + +def test_a_null_free_column_writes_no_sidecar_to_disk(tmp_path): + path = str(tmp_path / "clean.b2d") + make_table(with_nulls=False).save(path) + reopened = blosc2.CTable.open(path) + try: + assert not sidecar_key_exists(reopened, "a") + assert reopened._null_mask("a") is None + finally: + reopened.close() + + +def test_only_the_column_given_nulls_gets_a_sidecar(tmp_path): + path = str(tmp_path / "one.b2d") + make_table().save(path) + reopened = blosc2.CTable.open(path) + try: + assert sidecar_key_exists(reopened, "a") + assert not sidecar_key_exists(reopened, "c") + finally: + reopened.close() + + +# --------------------------------------------------------------------------- +# Chunk pinning +# --------------------------------------------------------------------------- + + +def test_sidecar_shares_its_column_row_grid(): + t = make_table() + col = t._cols["a"] + mask = t._null_mask("a") + assert mask.chunks[0] == col.chunks[0] + assert mask.blocks[0] == col.blocks[0] + + +def test_sidecar_grid_survives_a_chunk_override(tmp_path): + """``copy(chunks=…)`` reblocks the column; the sidecar must follow it.""" + t = make_table() + copied = t.copy(urlpath=str(tmp_path / "ovr.b2d"), chunks=16, blocks=8) + try: + assert copied._cols["a"].chunks[0] == 16 + assert copied._null_mask("a").chunks[0] == 16 + assert null_positions(copied) == list(NULL_POSITIONS) + finally: + copied.close() + + +def test_utf8_sidecar_falls_back_to_the_table_grid(): + """utf8 offsets carry ``n + 1`` entries, so they are not the grid to pin to.""" + Row = dataclasses.make_dataclass("Utf8Row", [("u", str, blosc2.field(blosc2.utf8(null_storage="mask")))]) + t = blosc2.CTable(Row, expected_size=64) + t.extend([(f"v{i}",) for i in range(64)]) + mask = t._ensure_null_mask("u") + assert mask.chunks[0] == t._valid_rows.chunks[0] + assert mask.blocks[0] == t._valid_rows.blocks[0] + + +# --------------------------------------------------------------------------- +# Persistence round-trips +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("suffix", [".b2d", ".b2z"]) +def test_sidecar_survives_save_and_open(tmp_path, suffix): + path = str(tmp_path / f"t{suffix}") + make_table().save(path) + reopened = blosc2.CTable.open(path) + try: + assert null_positions(reopened) == list(NULL_POSITIONS) + finally: + reopened.close() + + +def test_sidecar_survives_mmap_open(tmp_path): + path = str(tmp_path / "m.b2z") + make_table().save(path) + reopened = blosc2.CTable.open(path, mmap_mode="r") + try: + assert null_positions(reopened) == list(NULL_POSITIONS) + finally: + reopened.close() + + +def test_sidecar_survives_ctable_load(tmp_path): + path = str(tmp_path / "l.b2d") + make_table().save(path) + loaded = blosc2.CTable.load(path) + assert null_positions(loaded) == list(NULL_POSITIONS) + # load() rebuilds every array in RAM, so the pinning must be re-established + # rather than inherited. + assert loaded._null_mask("a").chunks[0] == loaded._cols["a"].chunks[0] + + +def test_sidecar_survives_cframe_round_trip(): + original = make_table() + rebuilt = blosc2.ctable_from_cframe(original.to_cframe()) + assert null_positions(rebuilt) == list(NULL_POSITIONS) + + +def test_cframe_of_a_null_free_table_carries_no_sidecar(): + rebuilt = blosc2.ctable_from_cframe(make_table(with_nulls=False).to_cframe()) + assert rebuilt._null_mask("a") is None + + +def test_sidecar_survives_inline_treestore(tmp_path): + path = str(tmp_path / "tree.b2d") + with blosc2.TreeStore(path, mode="w") as store: + store["/tbl"] = make_table() + with blosc2.TreeStore(path, mode="r") as store: + assert null_positions(store["/tbl"]) == list(NULL_POSITIONS) + + +def test_sidecar_survives_physical_pack_and_unpack(tmp_path): + """``to_b2z``/``to_b2d`` zip the leaves as-is; the sidecar is one of them.""" + src_path = str(tmp_path / "src.b2d") + make_table().save(src_path) + + packed = str(tmp_path / "packed.b2z") + src = blosc2.CTable.open(src_path, mode="r") + try: + src.to_b2z(packed) + finally: + src.close() + + zipped = blosc2.CTable.open(packed, mode="r") + try: + assert null_positions(zipped) == list(NULL_POSITIONS) + unpacked_path = str(tmp_path / "unpacked.b2d") + zipped.to_b2d(unpacked_path) + finally: + zipped.close() + + unpacked = blosc2.CTable.open(unpacked_path) + try: + assert null_positions(unpacked) == list(NULL_POSITIONS) + finally: + unpacked.close() + + +@pytest.mark.parametrize("compact", [True, False]) +def test_sidecar_survives_in_memory_copy(compact): + copied = make_table().copy(compact=compact) + assert null_positions(copied) == list(NULL_POSITIONS) + + +def test_copy_of_a_null_free_table_stays_sidecar_free(): + assert make_table(with_nulls=False).copy()._null_mask("a") is None + + +def test_view_materialization_remaps_the_sidecar(): + t = make_table() + materialized = t.where("b >= 5").copy() + # Physical rows 0-4 are filtered out, so physical nulls 7 and 41 land at + # logical 2 and 36; physical 3 is not in the view at all. + assert null_positions(materialized) == [2, 36] + + +def test_a_view_shares_its_base_sidecar_cache(): + t = make_table() + view = t.where("b >= 5") + assert view._null_masks is t._null_masks + assert view._null_mask("a") is t._null_mask("a") + + +# --------------------------------------------------------------------------- +# Capacity management +# --------------------------------------------------------------------------- + + +def test_grow_extends_the_sidecar_as_all_valid(): + t = make_table(n_rows=100, capacity=100) + before = len(t._valid_rows) + t.extend([(i, float(i), f"s{i}") for i in range(100, 150)]) + mask = t._null_mask("a") + assert mask.shape[0] == len(t._valid_rows) > before + # resize() zero-fills, i.e. *invalid*; the new tail must be corrected. + assert bool(np.asarray(mask[before:]).all()) + assert null_positions(t) == list(NULL_POSITIONS) + + +def test_repeated_grow_cycles_keep_the_sidecar_aligned(): + t = make_table(n_rows=20, capacity=20) + for start in range(20, 200, 20): + t.extend([(i, float(i), f"s{i}") for i in range(start, start + 20)]) + mask = t._null_mask("a") + assert mask.shape[0] == len(t._valid_rows) + assert null_positions(t) == [p for p in NULL_POSITIONS if p < 20] + assert int((~np.asarray(mask[: t._n_rows])).sum()) == 2 + + +def test_trim_capacity_shrinks_the_sidecar(): + t = make_table(n_rows=50, capacity=100) + t.trim_capacity() + mask = t._null_mask("a") + assert mask.shape == (50,) + assert null_positions(t) == list(NULL_POSITIONS) + + +def test_capacity_paths_find_an_unopened_sidecar(tmp_path): + """A reopened table has opened no sidecar; grow and trim must still find it. + + Iterating only the *materialized* sidecars would silently leave this one + at its old length, out of step with its column. + """ + path = str(tmp_path / "trim.b2d") + make_table(n_rows=50, capacity=100).save(path) + reopened = blosc2.CTable.open(path, mode="a") + try: + assert reopened._null_masks.materialized() == {} + reopened.extend([(i, float(i), f"s{i}") for i in range(50, 90)]) + assert reopened._null_mask("a").shape[0] == len(reopened._valid_rows) > 50 + reopened.trim_capacity() + assert reopened._null_mask("a").shape == (90,) + assert null_positions(reopened) == list(NULL_POSITIONS) + finally: + reopened.close() + + +def test_compact_gathers_the_sidecar_with_its_column(): + t = make_table() + t.delete(0) + t.compact() + # Every surviving row shifts down one, so the nulls do too. + assert null_positions(t) == [p - 1 for p in NULL_POSITIONS] + + +def test_compact_leaves_the_freed_tail_valid(): + t = make_table() + t.delete(0) + t.compact() + mask = np.asarray(t._null_mask("a")[:]) + assert bool(mask[t._n_rows :].all()) + + +# --------------------------------------------------------------------------- +# Column-level mutation +# --------------------------------------------------------------------------- + + +def test_drop_column_removes_the_sidecar_key(tmp_path): + path = str(tmp_path / "drop.b2d") + make_table().save(path) + table = blosc2.CTable.open(path, mode="a") + try: + key = table._storage._col_key("a") + _NOTNULL_SUFFIX + assert key in table._storage._open_store() + table.drop_column("a") + assert key not in table._storage._open_store() + assert table._null_mask_names == ["c"] + finally: + table.close() + + +@pytest.mark.parametrize("persistent", [True, False]) +def test_rename_column_carries_the_sidecar(tmp_path, persistent): + if persistent: + path = str(tmp_path / "ren.b2d") + make_table().save(path) + table = blosc2.CTable.open(path, mode="a") + else: + table = make_table() + try: + table.rename_column("a", "renamed") + assert null_positions(table, "renamed") == list(NULL_POSITIONS) + assert table._null_mask("a") is None + finally: + if persistent: + table.close() + + +def test_renamed_sidecar_survives_reopen(tmp_path): + path = str(tmp_path / "ren2.b2d") + make_table().save(path) + table = blosc2.CTable.open(path, mode="a") + try: + table.rename_column("a", "renamed") + finally: + table.close() + reopened = blosc2.CTable.open(path) + try: + assert null_positions(reopened, "renamed") == list(NULL_POSITIONS) + finally: + reopened.close() + + +# --------------------------------------------------------------------------- +# Storage-backend surface +# --------------------------------------------------------------------------- + + +def test_storage_reports_absence_before_creation(tmp_path): + path = str(tmp_path / "backend.b2d") + table = make_table(with_nulls=False, urlpath=path) + try: + assert table._storage.has_null_mask("a") is False + table._ensure_null_mask("a") + assert table._storage.has_null_mask("a") is True + finally: + table.close() + + +def test_delete_null_mask_is_a_no_op_when_absent(tmp_path): + path = str(tmp_path / "nodel.b2d") + table = make_table(with_nulls=False, urlpath=path) + try: + table._storage.delete_null_mask("a") # must not raise + assert table._storage.has_null_mask("a") is False + finally: + table.close() + + +def test_in_memory_storage_never_reports_a_stored_sidecar(): + """In-memory sidecars are held by the CTable, mirroring ``open_column``.""" + t = make_table() + assert t._storage.has_null_mask("a") is False + assert t._null_mask("a") is not None + + +def test_sidecar_key_is_beside_the_column_key(tmp_path): + path = str(tmp_path / "layout.b2d") + make_table().save(path) + storage = FileTableStorage(path, "r") + try: + keys = set(storage._open_store()) + assert "/_cols/a" in keys + assert "/_cols/a" + _NOTNULL_SUFFIX in keys + assert "/_cols/c" + _NOTNULL_SUFFIX not in keys + finally: + storage.close() + + +def test_sidecar_leaf_lands_on_disk(tmp_path): + path = str(tmp_path / "leaf.b2d") + make_table().save(path) + assert os.path.exists(os.path.join(path, "_cols", "a.notnull.b2nd")) From 800022c95e8f660c8653421188c3d300d897d97f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 11:46:36 +0200 Subject: [PATCH 05/47] Make mask columns readable and writable (mask-based-nulls 4) ``None`` becomes the canonical way to write a null into a fixed-width scalar column, which could not accept one at all before -- users had to write the sentinel literally. The null API now reads validity from the sidecar instead of inferring it from the values, which makes fillna() correct even when the fill equals data the column really holds. Two behaviours worth stating plainly: * Nullable bool is a real np.bool_ again. No uint8, no reserved 255. * NaN is a value, not a null. A mask-backed float column follows Arrow: only mask=False is missing. Sentinel float columns keep NaN-as-null forever, and the divergence is deliberate and tested on both sides. The riskiest part was not __setitem__, which needed one condition and one call per branch. It was that Column cached its NullChannel while the channel held the Column: a two-object cycle that refcounting can never break, harmless until extend() started building a channel per column, at which point every write pinned a whole table until the next gc pass. The cache is gone; the channel is a one-slot object and is rebuilt per access. Also not in the plan's site list: sort_by (all three forms), take, and slice each rebuild a table from gathered rows and silently dropped the sidecar. Factored into _permute_null_masks and _gather_null_masks_into, the latter leaving a copy sidecar-free when its selection contains no null. Co-Authored-By: Claude Opus 5 --- plans/mask-based-nulls.md | 48 ++- src/blosc2/ctable.py | 218 +++++++++++-- src/blosc2/ctable_nulls.py | 254 ++++++++++++++- src/blosc2/schema_vectorized.py | 26 +- tests/ctable/test_null_mask_api.py | 507 +++++++++++++++++++++++++++++ 5 files changed, 1012 insertions(+), 41 deletions(-) create mode 100644 tests/ctable/test_null_mask_api.py diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index f45e6d4b6..40fb21ab8 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,10 +1,11 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS — Phases 0–3 landed 2026-08-08.** Next up is Phase 4 (read/write + null -> API), the large, high-risk one. Two premises were disproven during implementation and are -> corrected in place, each in a blockquote beside the text it corrects: the index path cannot be -> fixed by a null-aware expression (§Expression layer), and the bool dtype-flip cannot move out of -> `__init__` (§Schema layer). Drafted 2026-08-08. +> **Status: IN PROGRESS — Phases 0–4 landed 2026-08-08.** Mask columns are now fully usable +> in memory and on disk; next up is Phase 5 (expressions + reductions), then Phase 6 +> (Arrow/Parquet). Two premises were disproven during implementation and are corrected in place, +> each in a blockquote beside the text it corrects: the index path cannot be fixed by a null-aware +> expression (§Expression layer), and the bool dtype-flip cannot move out of `__init__` +> (§Schema layer). Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -336,6 +337,39 @@ explicitly, coerce once, write values and mask together — and skip the fast pa re-add them in a follow-up. Threading mask writes through all six paths at once is where this project would break. +> **As built (2026-08-08).** `__setitem__` was not the hard part; the reroute was one added +> condition on the NDArray fast path plus one `_assign_validity` call per branch. Four things that +> were not in this section turned out to matter more: +> +> - **`NullChannel` must not be cached on its `Column`.** The pair was a two-object reference cycle +> (`Column._null_channel → NullChannel._col → Column`), and since a `Column` also holds its +> `CTable`, the moment `extend` started building a channel per column *every* write pinned a whole +> table until the next gc pass. Caught by `test_persistent_releases_without_gc`, which had been +> passing since long before this work. A weakref back to the column is the wrong fix — the channel +> is often the only owner (`table._null_channel(name)` builds a throwaway `Column`) — so the cache +> is gone instead and `_nulls` builds a fresh one-slot object per access. Pinned by +> `test_a_table_is_freed_without_a_gc_pass`. +> - **Validation runs before coercion**, so `schema_vectorized` and its ndarray branch had to learn +> that a bare `None` in the batch is a null for a mask column: a null cell has no value to +> constrain. Without this, `extend` raised out of `_validate_string_lengths` before reaching any +> of the new code. +> - **An overwrite replaces validity, it does not merge it.** `assign`/`__setitem__` must write +> `True` back over rows that were null before, which means `coerce_batch` returning `valid=None` +> ("nothing null in this batch") cannot simply be skipped there the way it can on append. +> `Column._assign_validity` is that distinction, and the one exception it keeps is a column with +> no sidecar at all, which is already all-valid. +> - **Every materialization path had to be found, not just the write paths.** `sort_by` (all three +> forms), `take`, `slice`, and `_sorted_small_copy_from_live_positions` each rebuild a table from +> gathered rows and silently dropped the sidecar. Factored into `_permute_null_masks` (in-place, +> shared with `compact`) and `_gather_null_masks_into` (copy). The latter skips columns whose +> gathered selection happens to be all-valid, so a copy that contains no null stays sidecar-free. +> +> Also landed here rather than in Phase 5: `null_pred`/`valid_pred` return the sidecar for mask +> columns. It is six lines inside `NullChannel`, and `_lazy_nonnull_mask` already consumes them, so +> leaving it out would have shipped aggregates that silently counted fill values. Fixed-shape +> ndarray columns still return `None` (an N-D values array does not broadcast against a +> one-flag-per-row sidecar) — that part stays Phase 5. + ### Null API — `ctable.py:2587-2721` `is_null()` (`:2627`) becomes `~channel.null_mask(...)`, i.e. O(1 byte/row) instead of @@ -627,8 +661,8 @@ default-created tables require them. | 1 | ✅ **Per-leaf null-predicate rewrite** *(storage-independent — pulled forward because it fixes sentinel tables that exist today)*. `_rewrite_null_predicates`, guards emitted inline and only where the sentinel could satisfy the leaf; validity pushed to the negation point. **`_exclude_null_positions` and the indexed-OR bail are retained** — see the correction above; an ordered index never evaluates the predicate, so a null-aware expression cannot fix it. Real payoff: string predicates become null-aware at all. | M | Med | | 2 | ✅ **Schema plumbing.** `_NullableSpecMixin`, `null_storage` kwarg on ~9 specs, conditional version 3, `NullPolicy.null_storage` (**still defaulting to `"sentinel"`**) with sentinel-field inference, `_resolved_null_storage` as the single decision point, `fill_value_for`, complex nullable (mask-only). Dtype-flip relocation **not** done — see the correction above. | S | Low | | 3 | ✅ **Storage sidecar.** 5 methods × 4 backends, lazy creation (absent key = all valid); `_grow`/`trim_capacity`/`compact`/`_save_to_storage`/`to_cframe`/`load`, **plus `copy()`'s in-memory path**, which the section above had missed; companion-suffix loop in delete/rename. `tests/ctable/test_null_persistence.py` (38 tests) drives it with a hand-built mask. | M | Low | -| 4 | **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`. Mask columns fully usable. | **L** | **High** (`__setitem__`) | -| 5 | **Expressions + reductions.** `_raw_null_pred`, `_lazy_nonnull_mask`, `_ndarray_values_for_reduction`, argmin/argmax, `_is_nullable_bool`. Includes the ndarray-propagation gain. | M | Med | +| 4 | ✅ **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`; **plus every gather-and-rebuild path** (`sort_by` ×3, `take`, `slice`), which this section had not listed. Mask columns fully usable. `tests/ctable/test_null_mask_api.py` (90 tests). | **L** | **High** (turned out to be the reference cycle, not `__setitem__`) | +| 5 | **Expressions + reductions.** `_raw_null_pred`, `_lazy_nonnull_mask`, `_ndarray_values_for_reduction`, argmin/argmax, `_is_nullable_bool`. Includes the ndarray-propagation gain. *(The base `null_pred`/`valid_pred` mask support landed in Phase 4 — see the note above.)* | M | Med | | 6 | **Arrow/Parquet.** Import + export for all V1 kinds, `packbits`/`unpackbits` LSB-first, `arrow_slice(validity=)`, delete the "no sentinel available" import error. Ships **opt-in** (`null_storage="mask"`); the default stays `"sentinel"`. | M | Med | | 7 | **Sort + groupby.** `_build_lex_keys`, `_sorted_positions_from_full_index` (big I/O win), `_utf8_rank_arrays(valid=)`, groupby `_null_mask` threading. | M | Med-High | | 8 | **Migration + docs.** `convert_nulls`, `Column.null_storage`, `info()`, `doc/reference/ctable.rst` null-policy rewrite, release notes. | S–M | Low | diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index ae1e2037f..ed4b702e4 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -1131,19 +1131,18 @@ def __init__(self, table: CTable, col_name: str, mask=None): self._table = table self._col_name = col_name self._mask = mask - self._null_channel = None @property def _nulls(self) -> NullChannel: """This column's validity channel (see :mod:`blosc2.ctable_nulls`). - Built on first use and kept for this ``Column``'s lifetime. The - channel reads through to the live schema on every access, so it stays - correct even if the column's spec is mutated in place. + Built fresh on each access and deliberately **not** cached: the channel + holds this ``Column``, and caching it here would close a reference + cycle that keeps the whole table alive until a gc pass. The channel is + a one-slot object and reads through to the live schema, so building one + is both cheap and always current. """ - if self._null_channel is None: - self._null_channel = NullChannel(self) - return self._null_channel + return NullChannel(self) @property def _raw_col(self): @@ -1264,6 +1263,20 @@ def _resolve_live_positions(self) -> np.ndarray: return slp return np.where(self._valid_rows[:])[0] + def _physical_index(self, key: int) -> int: + """Physical position of logical row *key*, honouring the view's order.""" + n_rows = len(self) + if key < 0: + key += n_rows + if not (0 <= key < n_rows): + raise IndexError(f"index {key} is out of bounds for column with size {n_rows}") + # A sorted view holds its positions in sorted, not physical-ascending, + # order, so the key-th entry is the answer directly. + cached = getattr(self._table, "_cached_live_positions", None) + if cached is not None and self._table.base is not None: + return int(cached[key]) + return int(_find_physical_index(self._valid_rows, key)) + def _has_identity_positions(self) -> bool: """True when logical row ``k`` maps to physical row ``k`` for every row. @@ -1494,10 +1507,13 @@ def __setitem__(self, key: int | slice | list | np.ndarray, value): # noqa: C90 if not (0 <= key < n_rows): raise IndexError(f"index {key} is out of bounds for column with size {n_rows}") pos_true = _find_physical_index(self._valid_rows, key) + value, is_valid = self._nulls.coerce_scalar(value) if self.is_ndarray: spec = self._table._schema.columns_by_name[self._col_name].spec value = CTable._coerce_ndarray_value(self._col_name, spec, value) self._raw_col[int(pos_true)] = value + if self._nulls.uses_mask and not (is_valid and self._nulls.valid_array() is None): + self._nulls.set_valid(int(pos_true), is_valid) elif isinstance(key, np.ndarray) and key.dtype == np.bool_: n_live = len(self) @@ -1507,6 +1523,7 @@ def __setitem__(self, key: int | slice | list | np.ndarray, value): # noqa: C90 ) all_pos = np.where(self._valid_rows[:])[0] phys_indices = all_pos[key] + value, valid = self._nulls.coerce_batch(value, len(phys_indices)) if self.is_list or self.is_varlen_scalar: if len(value) != len(phys_indices): raise ValueError("Length mismatch in list-column assignment") @@ -1519,6 +1536,7 @@ def __setitem__(self, key: int | slice | list | np.ndarray, value): # noqa: C90 elif isinstance(value, (list, tuple)): value = np.array(value, dtype=self._raw_col.dtype) self._raw_col[phys_indices] = value + self._assign_validity(phys_indices, valid) elif isinstance(key, (slice, list, tuple, np.ndarray)): # Fast path: slice of a blosc2.NDArray into a scalar or ndarray column when @@ -1533,6 +1551,11 @@ def __setitem__(self, key: int | slice | list | np.ndarray, value): # noqa: C90 and isinstance(value, blosc2.NDArray) and not self.is_list and not self.is_varlen_scalar + # Mask columns take the unified path below, which writes values + # and validity from one explicit phys_indices. Threading the + # sidecar through all the fast paths as well is a follow-up; + # correctness first. + and not self._nulls.uses_mask and _tbl.base is None and _tbl._resolve_last_pos() == _tbl._n_rows ): @@ -1573,6 +1596,7 @@ def _coerce(v, n): else: phys_indices = np.array([real_pos[i] for i in key], dtype=np.int64) + value, valid = self._nulls.coerce_batch(value, len(phys_indices)) if self.is_list or self.is_varlen_scalar: if len(value) != len(phys_indices): raise ValueError("Length mismatch in list-column assignment") @@ -1595,6 +1619,7 @@ def _coerce(v, n): self._raw_col[phys_indices[c:c_end]] = chunk else: self._raw_col[phys_indices] = value + self._assign_validity(phys_indices, valid) else: raise TypeError(f"Invalid index type: {type(key)}") @@ -1757,6 +1782,23 @@ def info(self) -> _CTableInfoReporter: """ return _CTableInfoReporter(self) + def _null_info_items(self, spec) -> list[tuple[str, object]]: + """The nullability rows of :attr:`info_items`. + + Beyond the plain flag, a nullable column reports *where* it keeps its + nulls, and a mask column also reports whether it has actually had to + write a sidecar yet — the visible difference between "nullable" and + "has ever held a null". + """ + nullable = self.null_value is not None or getattr(spec, "nullable", False) + items: list[tuple[str, object]] = [("nullable", nullable)] + if nullable: + channel = self._nulls + items.append(("null_storage", channel.kind)) + if channel.uses_mask: + items.append(("null_sidecar", channel.valid_array() is not None)) + return items + @property def info_items(self) -> list[tuple[str, object]]: """Structured summary items used by :attr:`info`.""" @@ -1812,7 +1854,7 @@ def info_items(self) -> list[tuple[str, object]]: if cratio is not None: items.append(("cratio", f"{cratio:.2f}x")) - items.append(("nullable", self.null_value is not None or getattr(spec, "nullable", False))) + items.extend(self._null_info_items(spec)) if self.is_dictionary: items.append(("dictionary_size", len(raw.dictionary))) @@ -2619,6 +2661,9 @@ def assign(self, data) -> None: root._mark_all_indexes_stale() return n_live = len(self) + # Split nullity off before the astype below, which has no way to + # represent a None in a typed array. + data, valid = self._nulls.coerce_batch(data, n_live) arr = np.asarray(data) if len(arr) != n_live: raise ValueError(f"assign() requires {n_live} values (live rows), got {len(arr)}.") @@ -2628,10 +2673,31 @@ def assign(self, data) -> None: raise TypeError(f"Cannot coerce data to column dtype {self.dtype!r}: {exc}") from exc live_pos = np.where(self._valid_rows[:])[0] self._raw_col[live_pos] = arr + # assign() replaces every live value, so validity is replaced wholesale + # too: rows this batch did not mark null must come back valid even if a + # previous assign had nulled them. + self._assign_validity(live_pos, valid) root = self._table._root_table root._mark_generated_columns_stale(self._col_name) root._mark_all_indexes_stale() + def _assign_validity(self, positions: np.ndarray, valid: np.ndarray | None) -> None: + """Overwrite the validity of *positions* after a wholesale value write. + + *valid* is what ``coerce_batch`` returned: ``None`` when nothing in the + batch was null. That still has to be *written* here rather than + skipped, because the rows may have been null before -- unlike an + append, an assign replaces what was there. A column with no sidecar is + the one exception: it is already all-valid. + """ + if not self._nulls.uses_mask: + return + if valid is None: + if self._nulls.valid_array() is None: + return + valid = True + self._nulls.set_valid(positions, valid) + def _assign_varlen_scalar(self, data) -> None: """``assign()`` for utf8/vlstring/vlbytes/struct/object columns. @@ -2640,8 +2706,9 @@ def _assign_varlen_scalar(self, data) -> None: varlen column rewrites its whole batch, so the loop would be quadratic. Dead slots keep their current contents. """ - values = list(data) n_live = len(self) + cells, valid = self._nulls.coerce_batch(list(data), n_live) + values = list(cells) if len(values) != n_live: raise ValueError(f"assign() requires {n_live} values (live rows), got {len(values)}.") raw = self._raw_col @@ -2649,12 +2716,14 @@ def _assign_varlen_scalar(self, data) -> None: n_phys = len(raw) if n_live == n_phys: raw.set_all(values) + self._assign_validity(np.arange(n_phys, dtype=np.intp), valid) return current = list(raw[:]) live_pos = np.flatnonzero(self._valid_rows[:n_phys]) for pos, value in zip(live_pos, values, strict=True): current[int(pos)] = value raw.set_all(current) + self._assign_validity(live_pos, valid) # ------------------------------------------------------------------ # Null sentinel support @@ -2709,14 +2778,39 @@ def fillna(self, value): Dictionary and variable-length scalar columns (whose nulls are native ``None`` cells) return a list; other columns return a NumPy array. + + Under mask storage this is correct even when *value* equals data the + column really holds, because which rows are null is read from the + sidecar rather than inferred from the values. A sentinel column + cannot make that distinction, by construction. """ if (self.is_dictionary or self.is_varlen_scalar) and not self.is_utf8: return [value if v is None else v for v in self[:]] arr = np.array(self[:], copy=True) - if self._nulls.kind == NULL_SENTINEL: + kind = self._nulls.kind + if kind == NULL_SENTINEL: arr[self._nulls.mask_for_values(arr)] = value + elif kind == NULL_MASK: + arr[self._nulls.null_mask()] = value return arr + def to_numpy(self, *, masked: bool = False): + """Return this column's live values as a NumPy array. + + Parameters + ---------- + masked: + When ``True``, return a :class:`numpy.ma.MaskedArray` whose mask + marks the null rows. Works for **both** null storages -- a + sentinel column derives the mask from its sentinel -- so callers + get one uniform way to ask for values-plus-validity regardless of + how the column happens to store its nulls. + """ + arr = np.asarray(self[:]) + if not masked: + return arr + return np.ma.MaskedArray(arr, mask=self.is_null()) + def _nonnull_chunks(self): """Yield chunks of live, non-null values. @@ -3470,13 +3564,10 @@ def __init__(self, table: CTable, prefix: str, leaves: list[str]): def _leaf_is_null_at_logical(self, leaf: str, idx: int) -> bool: col = self._table[leaf] - v = col[idx] - if col._nulls.kind != NULL_SENTINEL: - return v is None try: - return bool(col._nulls.mask_for_values(np.asarray([v]))[0]) + return col._nulls.is_null_at(idx) except Exception: - return v is None + return col[idx] is None def _row_value_at_logical(self, idx: int): # If every descendant leaf is null at this row, represent the struct as None. @@ -4961,11 +5052,23 @@ def _normalize_row_input(self, data: Any) -> dict[str, Any]: # Fallback: try positional indexing return {name: data[i] for i, name in enumerate(stored)} - def _coerce_row_to_storage(self, row: dict[str, Any]) -> dict[str, Any]: - """Coerce each value in *row* to the column's storage representation.""" + def _coerce_row_to_storage(self, row: dict[str, Any]) -> tuple[dict[str, Any], set[str]]: + """Coerce each value in *row* to the column's storage representation. + + Returns ``(storage_row, null_names)``, where *null_names* is the set of + mask-storage columns the caller must mark invalid. Their entry in + *storage_row* is this column's fill, so the write itself is an ordinary + typed one -- ``None`` never reaches the value array. + """ result = {} + null_names: set[str] = set() for col in self._schema.columns: val = row[col.name] + channel = self._null_channel(col.name) + if channel.uses_mask: + val, is_valid = channel.coerce_scalar(val) + if not is_valid: + null_names.add(col.name) if self._is_list_column(col): result[col.name] = coerce_list_cell(col.spec, val) elif self._is_varlen_scalar_column(col): @@ -4987,7 +5090,7 @@ def _coerce_row_to_storage(self, row: dict[str, Any]) -> dict[str, Any]: result[col.name] = np.array(val, dtype=col.dtype).item() else: result[col.name] = np.array(val, dtype=col.dtype).item() - return result + return result, null_names def _open_column_from_storage(self, storage: TableStorage, name: str): """Open one stored column from *storage*.""" @@ -5021,6 +5124,15 @@ def _null_masks(self) -> _NullMaskCache: self.__dict__["_null_mask_cache"] = cache return cache + def _null_channel(self, name: str) -> NullChannel: + """The null channel for column *name*. + + The physical write paths (``append``, ``extend``) have no ``Column`` of + their own; this gives them the same accessor the read paths use, over a + base-table column where logical and physical positions coincide. + """ + return self[name]._nulls + @property def _null_mask_names(self) -> list[str]: """Columns whose schema says their nulls live in a sidecar. @@ -5097,6 +5209,33 @@ def _existing_null_masks(self) -> dict[str, blosc2.NDArray]: found[name] = mask return found + def _permute_null_masks(self, positions: np.ndarray, n: int) -> None: + """Reorder every validity sidecar in place by *positions*, as the columns were. + + The tail past *n* becomes all-valid: those slots hold no live row, and + "valid" is what an absent sidecar would say of them. + """ + for mask in self._existing_null_masks().values(): + mask[:n] = mask[positions[:n]] + mask[n:] = True + + def _gather_null_masks_into(self, target: CTable, positions: np.ndarray, n: int) -> None: + """Gather this table's sidecars into *target*'s, under *positions*. + + A column with no sidecar here gets none there: nothing in it is null, + and materializing one in the copy would invent storage the source never + needed (decision 9). + """ + if n <= 0: + return + for name, mask in self._existing_null_masks().items(): + gathered = np.asarray(mask[positions]) + if gathered.all(): + # The selection happens to contain no null, so the copy needs + # no sidecar -- "absent" already says exactly this. + continue + target._ensure_null_mask(name)[:n] = gathered + def _drop_null_mask(self, name: str) -> None: """Forget and (for persistent tables) delete column *name*'s sidecar.""" self._null_masks.pop(name) @@ -6704,6 +6843,7 @@ def take(self, indices, /) -> CTable: else: result._cols[col_name][:n] = arr._take_numpy(physical_pos, axis=0) + self._gather_null_masks_into(result, physical_pos, n) result._valid_rows[:n] = True result._valid_rows[n:] = False result._n_rows = n @@ -11787,11 +11927,8 @@ def compact(self): end = min(end + block_size, self._n_rows) # Shuffle the validity sidecars by the same gather, so mask and values - # stay row-aligned. The freed tail becomes all-valid: those slots hold - # no live row, and "valid" is what an absent sidecar would say of them. - for mask in self._existing_null_masks().values(): - mask[: self._n_rows] = mask[real_poss[: self._n_rows]] - mask[self._n_rows :] = True + # stay row-aligned. + self._permute_null_masks(real_poss, self._n_rows) self._valid_rows[: self._n_rows] = True self._valid_rows[self._n_rows :] = False @@ -12344,6 +12481,7 @@ def _sorted_small_copy_from_live_positions( result._cols[col_name].codes[:n] = gathered[col_name][order] else: result._cols[col_name][:n] = gathered[col_name][order] + self._gather_null_masks_into(result, live_pos[order], n) result._valid_rows[:n] = True result._valid_rows[n:] = False result._n_rows = n @@ -12367,6 +12505,7 @@ def _sort_by_inplace(self, sorted_pos: np.ndarray, n: int) -> None: arr.set_all(arr[sorted_pos]) else: arr[:n] = arr[sorted_pos] + self._permute_null_masks(sorted_pos, n) self._valid_rows[:n] = True self._valid_rows[n:] = False self._n_rows = n @@ -12392,6 +12531,7 @@ def _sorted_copy_from_positions(self, sorted_pos: np.ndarray, n: int) -> CTable: result._cols[col_name].set_all(arr[sorted_pos]) else: result._cols[col_name][:n] = arr[sorted_pos] + self._gather_null_masks_into(result, sorted_pos, n) result._valid_rows[:n] = True result._valid_rows[n:] = False result._n_rows = n @@ -13008,7 +13148,7 @@ def append(self, data: list | np.void | np.ndarray) -> None: from blosc2.schema_validation import validate_row row = validate_row(self._schema, row) - row = self._coerce_row_to_storage(row) + row, null_names = self._coerce_row_to_storage(row) pos = self._resolve_last_pos() if pos >= len(self._valid_rows): @@ -13027,7 +13167,12 @@ def append(self, data: list | np.void | np.ndarray) -> None: if acc is not None and acc.valid: acc.feed(pos, np.asarray([row[name]], dtype=col_array.dtype)) + for name in null_names: + self._null_channel(name).set_valid(pos, False) + n_rows = self.nrows + # As in extend(): the row only becomes visible on the next line, so + # both the values and the validity above are written under its cover. self._valid_rows[pos] = True self._last_pos = pos + 1 self._n_rows = n_rows + 1 @@ -13190,16 +13335,30 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> list_processed_cols: dict[str, list] = {} varlen_scalar_processed_cols: dict[str, list] = {} dict_processed_cols: dict[str, list] = {} + # Validity for the mask-storage columns whose batch actually contained a + # null. Absent means "nothing was null", which needs no sidecar write. + batch_valid: dict[str, np.ndarray] = {} for name in current_col_names: col_meta = self._schema.columns_by_name[name] if self._is_list_column(col_meta): list_processed_cols[name] = list(raw_columns[name]) elif self._is_varlen_scalar_column(col_meta): - varlen_scalar_processed_cols[name] = list(raw_columns[name]) + channel = self._null_channel(name) + cells, valid = channel.coerce_batch(raw_columns[name], new_nrows) + if valid is not None: + batch_valid[name] = valid + varlen_scalar_processed_cols[name] = list(cells) elif self._is_dictionary_column(col_meta): dict_processed_cols[name] = list(raw_columns[name]) else: target_dtype = self._cols[name].dtype + # Split nullity out *before* NumPy sees the batch: a None in a + # float column would otherwise become a NaN, which under mask + # storage is a value (decision 6), not a null. + channel = self._null_channel(name) + raw_columns[name], valid = channel.coerce_batch(raw_columns[name], new_nrows) + if valid is not None: + batch_valid[name] = valid if isinstance(col_meta.spec, timestamp): values = np.asarray(raw_columns[name]) if np.issubdtype(values.dtype, np.datetime64): @@ -13267,7 +13426,16 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> self._cols[name][start_pos:end_pos] = values[:] self._feed_summary(name, start_pos, values) + # Validity sidecars, written before the rows go live below. Only the + # columns whose batch actually held a null appear here, so a null-free + # nullable column still never materializes a sidecar. + for name, valid in batch_valid.items(): + self._null_channel(name).set_valid(slice(start_pos, end_pos), valid) + n_rows = self.nrows + # Crash safety is inherited, not added: everything above -- values and + # validity alike -- is invisible until this line flips the rows live. + # Do not move any column write below it. self._valid_rows[start_pos:end_pos] = True self._last_pos = end_pos self._n_rows = n_rows + new_nrows diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index bbf6b3d86..dc8c99b1c 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -56,6 +56,7 @@ "NULL_SENTINEL", "NullChannel", "fill_value_for", + "is_na_marker", "is_nan_sentinel", "is_null_value", "kind_of_spec", @@ -83,6 +84,10 @@ def kind_of_spec(spec) -> str: return NULL_NONE if isinstance(spec, DictionarySpec): return NULL_CODE + if getattr(spec, "uses_mask", False): + # Checked before the native-None kinds so that a mask-storage utf8 + # column reports its sidecar rather than its container kind. + return NULL_MASK if isinstance(spec, _NATIVE_NULL_SPECS): return NULL_NATIVE # UTF8Spec is a variable-length kind but represents nulls with a sentinel @@ -92,6 +97,30 @@ def kind_of_spec(spec) -> str: return NULL_NONE +# Types whose instances mean "missing" without being ``None``. Matched by +# name so that neither pandas nor pyarrow has to be importable: ``pandas.NA`` +# is a ``NAType``, ``pyarrow.NA`` a ``NullScalar``, ``pandas.NaT`` a +# ``NaTType``. ``float('nan')`` is deliberately absent -- under mask storage +# NaN is a value, not a null (decision 6). +_NA_TYPE_NAMES = frozenset({"NAType", "NullScalar", "NaTType"}) + + +def is_na_marker(value) -> builtin_bool: + """True when *value* is a way of writing "this cell is null". + + ``None`` is the canonical spelling; the library NA singletons and a + ``datetime64`` ``NaT`` are accepted too, since each is unambiguously a + missing marker rather than a representable value. A float ``NaN`` is + **not** -- see :func:`~blosc2.schema.fill_value_for` and decision 6 of the + mask-storage design: keeping NaN a value is the point of a side channel. + """ + if value is None: + return True + if type(value).__name__ in _NA_TYPE_NAMES: + return True + return isinstance(value, np.datetime64) and builtin_bool(np.isnat(value)) + + def is_nan_sentinel(value) -> bool: """True when *value* is a NaN used as a null sentinel. @@ -347,14 +376,21 @@ def rewrite_null_predicates(expr: str, guards: dict[str, tuple[str, object]]) -> class NullChannel: """Uniform read accessor for one column's validity channel. - Subsumes the representations CTable uses -- in-band sentinel, dictionary - null code, native ``None``, and (once it lands) a sidecar validity array - -- so callers ask *what is null* without knowing which one a column uses. + Subsumes the four representations CTable uses -- sidecar validity array, + in-band sentinel, dictionary null code, native ``None`` -- so callers ask + *what is null* without knowing which one a column uses. Bound to a :class:`~blosc2.ctable.Column`, so it sees that column's view (sorted order, row filter) the same way the column itself does. Nothing is snapshotted: every property reads through to the live schema, which - keeps a cached channel correct across in-place spec mutation. + keeps a channel correct across in-place spec mutation. + + A channel holds its ``Column`` strongly and the ``Column`` does **not** + cache the channel back: the pair would otherwise be a reference cycle that + refcounting alone can never break, and since a ``Column`` also holds its + ``CTable``, every channel built on a write path would pin a whole table + until the next gc pass. Construction is one slot assignment, so rebuilding + a channel per access costs nothing worth caching. """ __slots__ = ("_col",) @@ -395,6 +431,63 @@ def null_code(self): """The reserved dictionary code, or ``None`` for the other kinds.""" return getattr(self.spec, "null_code", None) + @property + def uses_mask(self) -> builtin_bool: + """True when nullity lives in a sidecar validity array.""" + return self.kind == NULL_MASK + + @property + def fill_value(self): + """The value occupying this column's null slots under mask storage. + + Unobservable through the ``Column`` API and not part of the format + contract -- see :func:`~blosc2.schema.fill_value_for`. Widened to a + whole item for a fixed-shape ndarray column, whose null rows still + have to hold something of the right shape. + """ + base = fill_value_for(self.spec) + col = self._col + if col.is_ndarray: + return np.full(col.item_shape, base, dtype=col.dtype) + return base + + # ------------------------------------------------------------------ + # The sidecar (mask kind only) + # ------------------------------------------------------------------ + + def valid_array(self): + """The physical validity sidecar, or ``None`` when there is none. + + ``None`` for every non-mask kind, and also for a mask column that has + never been given a null: an absent sidecar means *all rows valid*, so + callers read that as never-null rather than as unknown. + """ + if self.kind != NULL_MASK: + return None + return self._col._table._null_mask(self._col._col_name) + + def _ensure_valid_array(self): + """The sidecar, materializing it if this is the column's first null.""" + return self._col._table._ensure_null_mask(self._col._col_name) + + def set_valid(self, key, valid) -> None: + """Record validity at *physical* positions *key*. + + A no-op for the non-mask kinds, whose nullity travels in band with the + values the caller has already written. + + Marking rows *valid* in a column with no sidecar is skipped rather + than made to materialize one: that is already what the column says. + """ + if self.kind != NULL_MASK: + return + arr = self.valid_array() + if arr is None: + if valid is True or (valid is not False and np.all(valid)): + return + arr = self._ensure_valid_array() + arr[key] = valid + # ------------------------------------------------------------------ # Reads # ------------------------------------------------------------------ @@ -413,17 +506,55 @@ def null_mask(self) -> np.ndarray: """True where this column's live values are null, one flag per live row.""" col = self._col kind = self.kind + if kind == NULL_MASK: + valid = self.valid_array() + if valid is None: + return np.zeros(len(col), dtype=np.bool_) + # Gathering at the live positions is what makes this honour the + # column's view -- sorted order and row filters alike. + return ~np.asarray(valid[col._resolve_live_positions()]) if kind == NULL_CODE: return col._dictionary_eq(None) if kind == NULL_NATIVE: return np.array([v is None for v in col], dtype=np.bool_) return self.mask_for_values(col[:]) + def is_null_at(self, index: int) -> builtin_bool: + """Whether the value at *logical* row *index* is null. + + Single-row form of :meth:`null_mask`, kept separate so the mask and + sentinel kinds can answer without materializing a whole column. + """ + col = self._col + kind = self.kind + if kind == NULL_NONE: + return False + if kind == NULL_MASK: + valid = self.valid_array() + if valid is None: + return False + return not builtin_bool(valid[int(col._physical_index(index))]) + if kind == NULL_SENTINEL: + return builtin_bool(self.mask_for_values(np.asarray([col[index]]))[0]) + return col[index] is None + def null_count(self) -> int: """Number of live rows that are null; ``0`` in O(1) when never null.""" kind = self.kind if kind == NULL_NONE: return 0 + if kind == NULL_MASK: + valid = self.valid_array() + if valid is None: + return 0 + col = self._col + if col._has_identity_positions(): + # Hole-free base table: count straight off the compressed + # sidecar. Bool NDArrays compress to almost nothing, so this + # is effectively O(chunks) rather than O(rows). + n = len(col) + return n - int(blosc2.count_nonzero(valid if n == valid.shape[0] else valid[:n])) + return int(self.null_mask().sum()) if kind == NULL_NATIVE: return sum(1 for v in self._col if v is None) return int(self.null_mask().sum()) @@ -431,6 +562,23 @@ def null_count(self) -> int: def nonnull_chunks(self): """Yield chunks of live values with the null ones removed.""" col = self._col + if self.kind == NULL_MASK: + valid = self.valid_array() + if valid is None: + yield from col.iter_chunks() + return + # Zip values against validity chunk for chunk. The sidecar shares + # the column's row grid (see CTable._null_mask_grid), so the two + # streams stay aligned without any re-chunking. + null = self.null_mask() + offset = 0 + for chunk in col.iter_chunks(): + keep = ~null[offset : offset + len(chunk)] + offset += len(chunk) + filtered = chunk[keep] + if len(filtered) > 0: + yield filtered + return sentinel = self.sentinel if sentinel is None: yield from col.iter_chunks() @@ -442,6 +590,82 @@ def nonnull_chunks(self): if len(filtered) > 0: yield filtered + # ------------------------------------------------------------------ + # Writes + # ------------------------------------------------------------------ + + def coerce_scalar(self, value): + """Split one incoming cell into ``(storage_value, is_valid)``. + + Under mask storage ``None`` becomes the canonical way to write a null. + Fixed-width scalar columns could not accept it at all before -- users + had to write the sentinel literally -- so this is new capability, not + re-plumbing. + """ + if self.kind != NULL_MASK or not is_na_marker(value): + return value, True + return self.fill_value, False + + def coerce_batch(self, values, n: int): + """Split an incoming batch into ``(storage_values, valid)``. + + *valid* is ``None`` when nothing in the batch was null -- the common + case, and the one that lets the caller skip the sidecar write entirely + and so never materialize one. Otherwise it is a length-*n* bool array + in Arrow polarity (``True`` = not null), and *storage_values* has this + column's fill substituted into the null slots, so what sits under + ``valid=False`` is deterministic rather than whatever NumPy made of a + ``None``. + + Null detection follows what the input is able to express: + + * ``np.ma.MaskedArray`` -- ``~arr.mask`` is the validity, verbatim; + * an object array or Python sequence -- ``None`` and the library NA + singletons are null (:func:`is_na_marker`); + * a ``datetime64`` array -- ``NaT`` is null; + * a float array -- **NaN is a value, not a null** (decision 6). A + mask column's whole point is that nullity lives outside the value + range, so nothing in a typed numeric array reads as missing. + """ + if self.kind != NULL_MASK: + return values, None + fill = self.fill_value + + if isinstance(values, np.ma.MaskedArray): + valid = ~np.ma.getmaskarray(values) + if valid.ndim > 1: # ndarray column: a row is null only if wholly masked + valid = valid.any(axis=tuple(range(1, valid.ndim))) + filled = np.ma.filled(values, fill) + return filled, (None if valid.all() else valid) + + if isinstance(values, blosc2.NDArray): + # Already in typed storage; there is no way for it to carry a None. + return values, None + + arr = values if isinstance(values, np.ndarray) else np.asarray(values, dtype=object) + if arr.dtype.kind == "M": + invalid = np.isnat(arr) + elif arr.dtype.kind == "O": + # Row-level: for an ndarray column each entry is a whole item, and + # only a wholesale None makes the row null. + invalid = np.fromiter((is_na_marker(v) for v in arr), dtype=np.bool_, count=len(arr)) + else: + # Typed numeric/bool/U/S input has no in-band way to say "null". + return values, None + + if not invalid.any(): + return values, None + out = np.asarray(arr, dtype=object).copy() + if isinstance(fill, np.ndarray): + # An ndarray column's fill is a whole item: assigning it through a + # boolean mask would broadcast its elements across the selected + # slots instead of storing one item in each. + for i in np.flatnonzero(invalid): + out[i] = fill + else: + out[invalid] = fill + return out.tolist(), ~invalid + # ------------------------------------------------------------------ # Lazy predicates over the raw physical array # ------------------------------------------------------------------ @@ -459,6 +683,9 @@ def null_pred(self): ``Column._ensure_queryable`` rejects them for arithmetic and comparisons before any predicate is built. """ + valid = self._mask_pred() + if valid is not None: + return ~valid col = self._col if col.is_ndarray: return None @@ -474,6 +701,9 @@ def valid_pred(self): Returns ``None`` under the same conditions as :meth:`null_pred`. """ + valid = self._mask_pred() + if valid is not None: + return valid col = self._col if col.is_ndarray: return None @@ -483,3 +713,19 @@ def valid_pred(self): if _is_nan(sentinel): return ~blosc2.isnan(col._raw_col) return col._raw_col != sentinel + + def _mask_pred(self): + """The sidecar as a physical validity operand, or ``None``. + + ``None`` covers both "not a mask column" and "a mask column with no + sidecar" -- the latter meaning never-null, which the expression layer + already handles by skipping the operand. + + Fixed-shape ndarray columns are excluded for now: their values array + is N-D while the sidecar is one flag per row, so the two do not + broadcast against each other in a lazy expression. + """ + if self.kind != NULL_MASK or self._col.is_ndarray: + return None + valid = self.valid_array() + return None if valid is None else blosc2.asarray(valid) diff --git a/src/blosc2/schema_vectorized.py b/src/blosc2/schema_vectorized.py index 5689246d4..550e48594 100644 --- a/src/blosc2/schema_vectorized.py +++ b/src/blosc2/schema_vectorized.py @@ -19,7 +19,7 @@ import numpy as np -from blosc2.ctable_nulls import sentinel_mask +from blosc2.ctable_nulls import is_na_marker, sentinel_mask from blosc2.list_array import _coerce_struct_item, coerce_list_cell from blosc2.schema import ListSpec, NDArraySpec, ObjectSpec, StructSpec from blosc2.schema_compiler import CompiledColumn, CompiledSchema # noqa: TC001 @@ -48,7 +48,18 @@ def _validate_string_lengths(col: CompiledColumn, arr: Any) -> None: def _null_mask_for_spec(arr: np.ndarray, spec) -> np.ndarray | None: - """Return a boolean mask True where values are the null sentinel, or None if no null_value.""" + """Return a boolean mask True where values are null, or ``None`` if none can be. + + A null cell has no value to constrain, so it must bypass the checks below + however the column spells it. Under sentinel storage that is the in-band + sentinel. Under mask storage validation runs *before* the batch is split + into values and validity, so what is still in the array is the caller's + ``None``/NA markers -- and only an object array can be carrying any. + """ + if getattr(spec, "uses_mask", False): + if arr.dtype.kind != "O" or arr.ndim != 1: + return None + return np.fromiter((is_na_marker(v) for v in arr), dtype=np.bool_, count=len(arr)) null_value = getattr(spec, "null_value", None) if null_value is None: return None @@ -83,12 +94,17 @@ def validate_column_values(col: CompiledColumn, values: Any) -> None: # noqa: C if isinstance(spec, ObjectSpec): return if isinstance(spec, NDArraySpec): - if getattr(spec, "null_value", None) is not None and not ( - isinstance(values, np.ndarray) and values.dtype != object - ): + uses_mask = getattr(spec, "uses_mask", False) + nullable = uses_mask or getattr(spec, "null_value", None) is not None + if nullable and not (isinstance(values, np.ndarray) and values.dtype != object): from blosc2.ctable import CTable for value in values: + # Under mask storage a null row is still a bare ``None`` here -- + # validation runs before the batch is split into values and + # validity -- and a null row has no item to shape-check. + if uses_mask and is_na_marker(value): + continue CTable._coerce_ndarray_value(col.name, spec, value) return arr = np.asarray(values, dtype=spec.dtype) diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py new file mode 100644 index 000000000..c2c72b880 --- /dev/null +++ b/tests/ctable/test_null_mask_api.py @@ -0,0 +1,507 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Reading and writing mask-storage nullable columns (Phase 4). + +Phase 3 gave the sidecar a place to live; this is where it becomes usable. +``None`` is now the canonical way to write a null into a fixed-width scalar +column -- which could not accept one at all before -- and the null API reads +validity from the sidecar rather than inferring it from the values. + +Two consequences are load-bearing and tested here: + +* **NaN is a value, not a null** (decision 6). A mask-backed float column + follows Arrow: only ``mask=False`` is missing. This is the whole point of + moving nullity to a side channel, and it is where mask and sentinel columns + deliberately diverge. +* **``fillna`` becomes correct when the fill collides with real data**, which + is impossible to get right under a sentinel. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 + + +def annotation_for(spec): + """An annotation the schema compiler accepts for *spec*. + + ``validate_annotation_matches_spec`` keys off ``spec.python_type`` for + everything except the two kinds that take ``object``. + """ + if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)): + return object + return spec.python_type + + +def row_type(**cols): + """A dataclass from ``name=spec`` pairs, annotations derived from the specs.""" + return dataclasses.make_dataclass( + "MaskRow", [(n, annotation_for(spec), blosc2.field(spec)) for n, spec in cols.items()] + ) + + +def table(rows, capacity=32, **cols): + t = blosc2.CTable(row_type(**cols), expected_size=capacity) + if rows: + t.extend(rows) + return t + + +def simple(values, spec=None, capacity=32): + """A one-column table of *values*, nulls written as ``None``.""" + spec = blosc2.int64(null_storage="mask") if spec is None else spec + return table([(v,) for v in values], capacity=capacity, a=spec) + + +# --------------------------------------------------------------------------- +# None is accepted, and nothing else is mistaken for it +# --------------------------------------------------------------------------- + +WRITEABLE_SPECS = [ + ("int64", blosc2.int64(null_storage="mask"), 7), + ("int8", blosc2.int8(null_storage="mask"), -128), + ("uint8", blosc2.uint8(null_storage="mask"), 255), + ("float64", blosc2.float64(null_storage="mask"), 1.5), + ("complex128", blosc2.complex128(null_storage="mask"), 1 + 2j), + ("bool", blosc2.bool(null_storage="mask"), True), + ("string", blosc2.string(max_length=4, null_storage="mask"), "abcd"), + ("bytes", blosc2.bytes(max_length=4, null_storage="mask"), b"abcd"), + ("utf8", blosc2.utf8(null_storage="mask"), "hello"), +] + + +@pytest.mark.parametrize(("label", "spec", "value"), WRITEABLE_SPECS) +def test_extend_accepts_none(label, spec, value): + t = simple([value, None, value], spec=spec) + assert t["a"].is_null().tolist() == [False, True, False] + assert t["a"].null_count() == 1 + + +@pytest.mark.parametrize(("label", "spec", "value"), WRITEABLE_SPECS) +def test_append_accepts_none(label, spec, value): + t = simple([value], spec=spec) + t.append((None,)) + assert t["a"].is_null().tolist() == [False, True] + + +@pytest.mark.parametrize(("label", "spec", "value"), WRITEABLE_SPECS) +def test_setitem_accepts_none(label, spec, value): + t = simple([value, value], spec=spec) + t["a"][1] = None + assert t["a"].is_null().tolist() == [False, True] + + +@pytest.mark.parametrize(("label", "spec", "value"), WRITEABLE_SPECS) +def test_assign_accepts_none(label, spec, value): + t = simple([value, value], spec=spec) + t["a"].assign([None, value]) + assert t["a"].is_null().tolist() == [True, False] + + +def test_nullable_bool_is_a_real_bool(): + """The case that motivated the whole design: no uint8, no reserved 255.""" + t = simple([True, None, False], spec=blosc2.bool(null_storage="mask")) + assert t["a"].dtype == np.dtype(np.bool_) + assert t["a"][:].tolist() == [True, False, False] + assert t["a"].is_null().tolist() == [False, True, False] + + +def test_int8_keeps_its_full_range_alongside_nulls(): + values = list(range(-128, 128)) + t = simple([*values, None], spec=blosc2.int8(null_storage="mask"), capacity=300) + assert t["a"].dtype == np.dtype(np.int8) + assert t["a"][:-1].tolist() == values + assert t["a"].is_null()[-1] + assert t["a"].null_count() == 1 + + +def test_string_keeps_its_declared_width(): + """No sentinel means no max_length widening to fit one.""" + t = simple(["abcd", None], spec=blosc2.string(max_length=4, null_storage="mask")) + assert t["a"].dtype == np.dtype("U4") + + +def test_utf8_accepts_text_no_sentinel_could_survive(): + tricky = ["", "\x00", "__BLOSC2_NULL__", "🎉x"] + t = simple([*tricky, None], spec=blosc2.utf8(null_storage="mask")) + assert list(t["a"][:-1]) == tricky + assert t["a"].is_null().tolist() == [False] * 4 + [True] + + +def test_timestamp_accepts_none(): + spec = blosc2.timestamp(null_storage="mask") + t = simple(["2020-01-01", None, "2020-01-03"], spec=spec) + assert t["a"].is_null().tolist() == [False, True, False] + assert np.isnat(t["a"][:][1]) + + +def test_ndarray_column_accepts_none(): + spec = blosc2.ndarray((3,), dtype=blosc2.float32(), null_storage="mask") + item = np.ones(3, dtype=np.float32) + t = simple([item, None, item * 2], spec=spec) + assert t["a"].is_null().tolist() == [False, True, False] + assert t["a"][:][0].tolist() == [1.0, 1.0, 1.0] + + +# --------------------------------------------------------------------------- +# Decision 6: NaN is a value +# --------------------------------------------------------------------------- + + +def test_nan_is_a_value_not_a_null(): + t = simple([1.0, float("nan"), None], spec=blosc2.float64(null_storage="mask")) + assert t["a"].is_null().tolist() == [False, False, True] + assert t["a"].null_count() == 1 + assert np.isnan(t["a"][:][1]) + + +def test_sentinel_float_still_treats_nan_as_null(): + """The two storages diverge here on purpose, and both are documented.""" + t = simple([1.0, float("nan")], spec=blosc2.float64(null_value=float("nan"))) + assert t["a"].is_null().tolist() == [False, True] + + +def test_signed_zero_and_inf_are_values(): + values = [0.0, -0.0, float("inf"), float("-inf")] + t = simple([*values, None], spec=blosc2.float64(null_storage="mask")) + assert t["a"].is_null().tolist() == [False] * 4 + [True] + assert np.array_equal(t["a"][:-1], np.array(values), equal_nan=True) + + +def test_int64_min_is_a_value_in_a_timestamp_column(): + spec = blosc2.timestamp(null_storage="mask") + t = simple([np.datetime64(np.iinfo(np.int64).min + 1, "us"), None], spec=spec) + assert t["a"].is_null().tolist() == [False, True] + + +# --------------------------------------------------------------------------- +# Input forms that carry their own validity +# --------------------------------------------------------------------------- + + +def test_masked_array_input_supplies_validity_verbatim(): + spec = blosc2.float64(null_storage="mask") + data = np.ma.MaskedArray([1.0, 2.0, np.nan], mask=[False, True, False]) + t = blosc2.CTable(row_type(a=spec), expected_size=8) + t.extend({"a": data}) + # The NaN is *not* masked, so it stays a value; only index 1 is null. + assert t["a"].is_null().tolist() == [False, True, False] + + +def test_numpy_float_array_input_has_no_nulls(): + spec = blosc2.float64(null_storage="mask") + t = blosc2.CTable(row_type(a=spec), expected_size=8) + t.extend({"a": np.array([1.0, np.nan, 3.0])}) + assert t["a"].null_count() == 0 + assert t["a"].is_null().tolist() == [False, False, False] + + +def test_nat_reads_as_null_in_a_timestamp_column(): + spec = blosc2.timestamp(null_storage="mask") + t = simple([np.datetime64("2020-01-01"), np.datetime64("NaT")], spec=spec) + assert t["a"].is_null().tolist() == [False, True] + + +# --------------------------------------------------------------------------- +# Decision 9 through the public API +# --------------------------------------------------------------------------- + + +def test_a_null_free_batch_writes_no_sidecar(): + t = simple([1, 2, 3]) + assert t._null_mask("a") is None + assert t["a"].null_count() == 0 + assert t["a"].is_null().tolist() == [False, False, False] + + +def test_the_sidecar_appears_on_the_first_null(): + t = simple([1, 2, 3]) + assert t._null_mask("a") is None + t.append((None,)) + assert t._null_mask("a") is not None + + +def test_marking_a_row_valid_does_not_create_a_sidecar(): + t = simple([1, 2, 3]) + t["a"][0] = 5 + assert t._null_mask("a") is None + + +def test_info_reports_storage_and_whether_a_sidecar_exists(): + t = simple([1, 2, 3]) + items = dict(t["a"].info_items) + assert items["nullable"] is True + assert items["null_storage"] == "mask" + assert items["null_sidecar"] is False + t.append((None,)) + assert dict(t["a"].info_items)["null_sidecar"] is True + + +def test_non_nullable_column_reports_no_storage(): + t = table([(1,)], a=blosc2.int64()) + assert "null_storage" not in dict(t["a"].info_items) + + +# --------------------------------------------------------------------------- +# Overwrite semantics: a write replaces validity, it does not merge it +# --------------------------------------------------------------------------- + + +def test_assign_clears_previously_written_nulls(): + t = simple([1, None, 3]) + t["a"].assign([1, 2, 3]) + assert t["a"].null_count() == 0 + assert t["a"][:].tolist() == [1, 2, 3] + + +def test_setitem_clears_a_null(): + t = simple([1, None, 3]) + t["a"][1] = 2 + assert t["a"].null_count() == 0 + assert t["a"][:].tolist() == [1, 2, 3] + + +def test_setitem_slice_replaces_validity(): + t = simple([1, None, 3, None]) + t["a"][0:3] = [None, 2, 3] + assert t["a"].is_null().tolist() == [True, False, False, True] + + +def test_setitem_boolean_mask_replaces_validity(): + t = simple([1, None, 3, 4]) + t["a"][np.array([True, True, False, False])] = [None, 2] + assert t["a"].is_null().tolist() == [True, False, False, False] + + +def test_setitem_index_list_replaces_validity(): + t = simple([1, None, 3, 4]) + t["a"][[1, 3]] = [2, None] + assert t["a"].is_null().tolist() == [False, False, False, True] + + +# --------------------------------------------------------------------------- +# The null API +# --------------------------------------------------------------------------- + + +def test_notnull_is_the_complement_of_is_null(): + t = simple([1, None, 3]) + assert (t["a"].notnull() == ~t["a"].is_null()).all() + + +def test_fillna_is_correct_when_the_fill_collides_with_real_data(): + """Impossible under a sentinel: there, 7 *is* how a null would look.""" + t = simple([7, None, 3]) + assert t["a"].fillna(7).tolist() == [7, 7, 3] + # ...and the column itself is unchanged, still one null. + assert t["a"].null_count() == 1 + + +def test_to_numpy_masked_marks_only_the_nulls(): + t = simple([1.0, float("nan"), None], spec=blosc2.float64(null_storage="mask")) + out = t["a"].to_numpy(masked=True) + assert isinstance(out, np.ma.MaskedArray) + assert out.mask.tolist() == [False, False, True] + + +def test_to_numpy_masked_works_for_sentinel_columns_too(): + """One uniform way to ask for values-plus-validity, whatever the storage.""" + t = simple([1, -1, 3], spec=blosc2.int64(null_value=-1)) + out = t["a"].to_numpy(masked=True) + assert out.mask.tolist() == [False, True, False] + + +def test_to_numpy_without_masked_is_a_plain_array(): + t = simple([1, None, 3]) + assert isinstance(t["a"].to_numpy(), np.ndarray) + assert not isinstance(t["a"].to_numpy(), np.ma.MaskedArray) + + +def test_unique_and_value_counts_skip_nulls(): + t = simple([1, None, 3, 1]) + assert t["a"].unique().tolist() == [1, 3] + assert t["a"].value_counts() == {1: 2, 3: 1} + + +def test_dropna_uses_the_sidecar(): + t = table( + [(1, 10), (None, 20), (3, 30)], + a=blosc2.int64(null_storage="mask"), + b=blosc2.int64(), + ) + assert t.dropna()["b"][:].tolist() == [10, 30] + + +def test_reductions_skip_nulls(): + t = simple([2.0, None, 4.0], spec=blosc2.float64(null_storage="mask")) + assert t["a"].sum() == 6.0 + assert t["a"].mean() == 3.0 + + +def test_null_count_with_deletions(): + t = simple([None, 1, None, 3]) + t.delete(0) + assert t["a"].null_count() == 1 + assert t["a"].is_null().tolist() == [False, True, False] + + +# --------------------------------------------------------------------------- +# Views keep their nulls +# --------------------------------------------------------------------------- + + +def two_col_table(): + return table( + [(None if i in (1, 4) else i, 10 - i) for i in range(6)], + a=blosc2.int64(null_storage="mask"), + b=blosc2.int64(), + ) + + +def test_where_view_remaps_nulls(): + t = two_col_table() + assert t.where("b < 8")["a"].is_null().tolist() == [False, True, False] + + +def test_slice_view_remaps_nulls(): + assert two_col_table()[1:5]["a"].is_null().tolist() == [True, False, False, True] + + +def test_reversed_slice_view_remaps_nulls(): + assert two_col_table()[::-1]["a"].is_null().tolist() == [False, True, False, False, True, False] + + +@pytest.mark.parametrize("view", [True, False]) +def test_sort_by_keeps_nulls_with_their_rows(view): + sorted_t = two_col_table().sort_by("b", view=view) + assert sorted_t["a"].is_null().tolist() == [False, True, False, False, True, False] + assert sorted_t["b"][:].tolist() == [5, 6, 7, 8, 9, 10] + + +def test_sort_by_inplace_keeps_nulls_with_their_rows(): + t = two_col_table() + t.sort_by("b", inplace=True) + assert t["a"].is_null().tolist() == [False, True, False, False, True, False] + + +def test_sorting_a_view_keeps_nulls_with_their_rows(): + t = two_col_table() + sorted_view = t.where("b > 5").sort_by("b") + assert sorted_view["a"].is_null().tolist() == [True, False, False, True, False] + + +def test_take_keeps_nulls_with_their_rows(): + assert two_col_table().take([4, 1, 0])["a"].is_null().tolist() == [True, True, False] + + +def test_slice_copy_keeps_nulls_with_their_rows(): + assert two_col_table().slice(1, 5)["a"].is_null().tolist() == [True, False, False, True] + + +@pytest.mark.parametrize("compact", [True, False]) +def test_copy_keeps_nulls(compact): + assert two_col_table().copy(compact=compact)["a"].null_count() == 2 + + +def test_compact_keeps_nulls_with_their_rows(): + t = two_col_table() + t.delete(0) + t.compact() + assert t["a"].is_null().tolist() == [True, False, False, True, False] + + +def test_a_view_of_a_null_free_column_stays_sidecar_free(): + t = simple([1, 2, 3, 4]) + assert t.take([0, 2])._null_mask("a") is None + + +def test_gathering_only_valid_rows_needs_no_sidecar(): + """Decision 9 again: the copy has no nulls, so it gets no bytes.""" + t = two_col_table() + assert t.take([0, 2, 3])._null_mask("a") is None + + +# --------------------------------------------------------------------------- +# Persistence of nulls written through the API +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("suffix", [".b2d", ".b2z"]) +def test_nulls_survive_save_and_reopen(tmp_path, suffix): + path = str(tmp_path / f"t{suffix}") + two_col_table().save(path) + reopened = blosc2.CTable.open(path) + try: + assert reopened["a"].is_null().tolist() == [False, True, False, False, True, False] + assert reopened["a"].null_count() == 2 + finally: + reopened.close() + + +def test_nulls_survive_a_cframe_round_trip(): + rebuilt = blosc2.ctable_from_cframe(two_col_table().to_cframe()) + assert rebuilt["a"].is_null().tolist() == [False, True, False, False, True, False] + + +def test_appending_to_a_reopened_table_keeps_earlier_nulls(tmp_path): + path = str(tmp_path / "grow.b2d") + two_col_table().save(path) + reopened = blosc2.CTable.open(path, mode="a") + try: + reopened.append((None, 99)) + assert reopened["a"].is_null().tolist() == [ + False, True, False, False, True, False, True, + ] # fmt: skip + finally: + reopened.close() + + +# --------------------------------------------------------------------------- +# Sentinel columns are untouched +# --------------------------------------------------------------------------- + + +def test_sentinel_columns_keep_their_behaviour(): + t = simple([1, -1, 3], spec=blosc2.int64(null_value=-1)) + assert t["a"].null_storage == "sentinel" + assert t["a"].is_null().tolist() == [False, True, False] + assert t._null_mask("a") is None + + +def test_sentinel_bool_is_still_uint8_backed(): + t = simple([True, 255, False], spec=blosc2.bool(nullable=True, null_value=255)) + assert t["a"].dtype == np.dtype(np.uint8) + assert t["a"].is_null().tolist() == [False, True, False] + + +def test_channel_is_not_cached_on_its_column(): + """A cached channel would close a Column-CTable reference cycle.""" + col = simple([1, 2])["a"] + assert col._nulls is not col._nulls + assert col._nulls.kind == "mask" + + +def test_a_table_is_freed_without_a_gc_pass(): + """The regression the no-caching rule above exists to prevent.""" + import gc + import weakref + + def build(): + return weakref.ref(simple([1, None, 3])) + + gc.disable() + try: + ref = build() + assert ref() is None + finally: + gc.enable() From 6a26836d3c2dd35f3f3cc0e1afc73b579947da8c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 11:59:42 +0200 Subject: [PATCH 06/47] Stop reducing mask columns over their fill (mask-based-nulls 5) argmin/argmax and the ndarray reduction path both keyed off ``null_value is not None``, so under mask storage they reduced over the fill. For an int column the fill is 0 -- a plausible-looking minimum for positive data, and a plausible-looking maximum for negative data. ``_reduction_null_mask`` is now the one place that answers "which live rows are null", from the sentinel for one storage and from the sidecar for the other. The expression layer needed nothing: Phases 0-4 had already left it consuming an opaque boolean predicate, and Phase 4 taught the channel to produce one from the sidecar. Two things the plan expected here are *not* done, because measuring them showed the premises were wrong. Both are now pinned by tests and explained at the site where someone would otherwise try them again: * Fixed-shape ndarray columns get no lazy null predicate. Row-level is not enough -- blosc2.where(pred_(n,1), nan, values_(n,3)) returns (n,1), silently dropping the item dimension from the values, and combining an (n,1) predicate with the (n,) row mask NullableExpr reductions use explodes to (n,n). They get their null handling from the NumPy reduction paths instead. * The summary-index min/max shortcut stays disabled for mask columns. Letting them through on the grounds that their fill is NaN is defeated by NaN being a *value* under a mask: on [1.0, nan, 5.0, null, 3.0] the scan gives nan while the summaries would answer 1.0/5.0, so the same query would depend on whether an index exists. Co-Authored-By: Claude Opus 5 --- plans/mask-based-nulls.md | 41 ++- src/blosc2/ctable.py | 44 ++- src/blosc2/ctable_nulls.py | 14 +- tests/ctable/test_null_mask_expressions.py | 327 +++++++++++++++++++++ 4 files changed, 408 insertions(+), 18 deletions(-) create mode 100644 tests/ctable/test_null_mask_expressions.py diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 40fb21ab8..39b9ad13a 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,11 +1,13 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS — Phases 0–4 landed 2026-08-08.** Mask columns are now fully usable -> in memory and on disk; next up is Phase 5 (expressions + reductions), then Phase 6 -> (Arrow/Parquet). Two premises were disproven during implementation and are corrected in place, -> each in a blockquote beside the text it corrects: the index path cannot be fixed by a null-aware -> expression (§Expression layer), and the bool dtype-flip cannot move out of `__init__` -> (§Schema layer). Drafted 2026-08-08. +> **Status: IN PROGRESS — Phases 0–5 landed 2026-08-08.** Mask columns are fully usable in memory +> and on disk, and correct through expressions and reductions; next up is Phase 6 (Arrow/Parquet), +> which is what the whole design is *for*. Four premises were disproven during implementation and +> are corrected in place, each in a blockquote beside the text it corrects: the index path cannot +> be fixed by a null-aware expression (§Expression layer), the bool dtype-flip cannot move out of +> `__init__` (§Schema layer), ndarray columns do not get lazy null propagation for free +> (§Expression layer), and the "free" summary min/max fast path is unsound (§Reductions). +> Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -399,6 +401,20 @@ gain null propagation in expressions for free**. And `_lazy_nonnull_mask` (`:277 stored array instead of synthesizing a comparison, keeping the miniexpr reduction fast path with one fewer computed operand. +> **Correction (measured 2026-08-08).** The second gain is real and landed. **The first is not, +> and was not done.** Row-level is necessary but not sufficient: the values array is +> `(n, *item_shape)` and the sidecar is `(n,)`, and reshaping it to `(n, 1, …)` — the obvious fix — +> fails twice. `blosc2.where(pred_(n,1), nan, values_(n,3))` returns shape `(n, 1)`, silently +> **dropping the item dimension from the values** rather than broadcasting; and `NullableExpr`'s +> reduction mask combines the predicate with the `(n,)` `_valid_rows`, where `(n,) & (n,1)` explodes +> to `(n, n)`. Row-level null propagation for ndarray columns needs broadcasting support in the lazy +> layer, not a reshape at this call site. +> +> What ndarray columns *do* gain in this phase is null-aware **reductions**, which are NumPy-based +> and row-level throughout: `min`/`max`/`sum`/`mean`/`argmin`/`argmax` now skip null rows instead of +> reducing over the fill item. Pinned by `test_ndarray_columns_still_get_no_lazy_null_predicate`, +> which asserts both halves — no predicate, working reduction — so the gap stays deliberate. + `_is_nullable_bool` (`:1824-1831`) becomes `kind == "bool" and channel.kind == NULL_SENTINEL`; the `raw_col == 1` rewrites (`:2039`, `:2049`, `:2767`, `:13392`) go dead for mask bools and stay alive forever for sentinel ones. @@ -458,6 +474,17 @@ never consults a side channel. Two honest routes: 1. *Free, partial*: with the NaN float fill (decision 5), mask-backed float columns qualify under the existing `is_nan_float` escape hatch at `:3045` with a one-condition change. Floats only — `int64.min` **is** the block minimum, so timestamps get nothing free. + + > **Correction (measured 2026-08-08). Route 1 is unsound and was not taken.** It is defeated by + > decision 6, three sections up: the summary builder drops NaNs, and under mask storage a NaN is + > a *value*, so it drops real data too. On `[1.0, nan, 5.0, null, 3.0]` the scan gives `nan` + > (NumPy semantics, NaN participates) while the summaries would answer `1.0`/`5.0` — the same + > query returning different answers depending on whether an index happens to exist. Contrast the + > sentinel-NaN column the hatch was written for, where NaN *is* the null, so dropping it is + > exactly right and the two paths agree. Mask columns keep the bail; the reasoning is now a + > comment at the bail site so nobody re-derives the one-liner. Both halves pinned by + > `test_summary_minmax_shortcut_stays_disabled_for_mask_columns` and + > `test_sentinel_nan_float_keeps_its_summary_shortcut`. 2. *Real fix (Phase 10)*: make the summary builder in `ctable_indexing.py` mask-aware — extrema over `values[valid]`, a per-segment `all_null` flag, `"null_aware": true` in the descriptor. This retroactively enables the fast path for **sentinel** columns too. @@ -662,7 +689,7 @@ default-created tables require them. | 2 | ✅ **Schema plumbing.** `_NullableSpecMixin`, `null_storage` kwarg on ~9 specs, conditional version 3, `NullPolicy.null_storage` (**still defaulting to `"sentinel"`**) with sentinel-field inference, `_resolved_null_storage` as the single decision point, `fill_value_for`, complex nullable (mask-only). Dtype-flip relocation **not** done — see the correction above. | S | Low | | 3 | ✅ **Storage sidecar.** 5 methods × 4 backends, lazy creation (absent key = all valid); `_grow`/`trim_capacity`/`compact`/`_save_to_storage`/`to_cframe`/`load`, **plus `copy()`'s in-memory path**, which the section above had missed; companion-suffix loop in delete/rename. `tests/ctable/test_null_persistence.py` (38 tests) drives it with a hand-built mask. | M | Low | | 4 | ✅ **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`; **plus every gather-and-rebuild path** (`sort_by` ×3, `take`, `slice`), which this section had not listed. Mask columns fully usable. `tests/ctable/test_null_mask_api.py` (90 tests). | **L** | **High** (turned out to be the reference cycle, not `__setitem__`) | -| 5 | **Expressions + reductions.** `_raw_null_pred`, `_lazy_nonnull_mask`, `_ndarray_values_for_reduction`, argmin/argmax, `_is_nullable_bool`. Includes the ndarray-propagation gain. *(The base `null_pred`/`valid_pred` mask support landed in Phase 4 — see the note above.)* | M | Med | +| 5 | ✅ **Expressions + reductions.** `_ndarray_values_for_reduction`, argmin/argmax (both were reducing over the *fill*), `_reduction_null_mask` as the one storage-agnostic entry point. `_raw_null_pred`/`_lazy_nonnull_mask`/`_is_nullable_bool` needed nothing — Phases 0–4 had already made them storage-agnostic. **The ndarray-propagation gain is not real and was not done**, and the free summary fast path is unsound; both corrections are above. `tests/ctable/test_null_mask_expressions.py` (39 tests). | S (was M) | Low (was Med) | | 6 | **Arrow/Parquet.** Import + export for all V1 kinds, `packbits`/`unpackbits` LSB-first, `arrow_slice(validity=)`, delete the "no sentinel available" import error. Ships **opt-in** (`null_storage="mask"`); the default stays `"sentinel"`. | M | Med | | 7 | **Sort + groupby.** `_build_lex_keys`, `_sorted_positions_from_full_index` (big I/O win), `_utf8_rank_arrays(valid=)`, groupby `_null_mask` threading. | M | Med-High | | 8 | **Migration + docs.** `convert_nulls`, `Column.null_storage`, `info()`, `doc/reference/ctable.rst` null-policy rewrite, release notes. | S–M | Low | diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index ed4b702e4..2ec6a8d40 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -2964,9 +2964,25 @@ def _sum_lazy_fastpath(self, acc_dtype, where=None, *, jit=None, jit_backend=Non except Exception: return NotImplemented + def _reduction_null_mask(self, arr: np.ndarray) -> np.ndarray | None: + """Per-live-row null flags for already-materialized values *arr*. + + ``None`` when this column can hold no nulls at all, which lets the + caller skip the filtering entirely. A sentinel column answers from + *arr* itself; a mask column reads its sidecar, which is why this cannot + simply be :meth:`_null_mask_for`. Both answers are one flag per row, + including for a fixed-shape ndarray column. + """ + channel = self._nulls + if channel.uses_mask: + return channel.null_mask() + if self.null_value is None: + return None + return channel.mask_for_values(arr) + def _ndarray_values_for_reduction(self, where=None) -> np.ndarray: arr = np.asarray(self[:]) - null_mask = self._null_mask_for(arr) if self.null_value is not None else None + null_mask = self._reduction_null_mask(arr) if null_mask is not None and null_mask.any(): arr = arr[~null_mask] if where is None: @@ -3168,7 +3184,19 @@ def _summary_minmax_source(self): null_value = getattr(spec, "null_value", None) is_nan_float = dtype.kind == "f" and is_nan_sentinel(null_value) if nullable and not is_nan_float: - return None # non-NaN sentinel leaks into the block extrema + # A non-NaN sentinel leaks into the block extrema. + # + # It is tempting to let mask-backed float columns through here too, + # on the grounds that their fill is NaN and the summary builder + # drops NaNs. That is wrong, and precisely *because* NaN is a value + # in a mask column (Arrow semantics): a genuine NaN poisons the + # scanned min()/max() to NaN, while the summary silently drops it + # and answers with a real extremum. Measured on + # ``[1.0, nan, 5.0, null, 3.0]``: scan gives nan, summaries would + # give 1.0/5.0. Making the index null-aware is the real fix + # (plans/mask-based-nulls.md, phase 10); until then mask columns + # take the same bail as sentinel ones. + return None root = table._root_table desc = root._get_index_catalog().get(self._col_name) if not desc or desc.get("stale", False): @@ -3358,9 +3386,9 @@ def argmin(self, axis=None, *, where=None): arr = np.asarray(self[:]) if arr.size == 0: raise ValueError("argmin() called on an empty column.") - mask = ( - self._null_mask_for(arr) if self.null_value is not None else np.zeros(len(arr), dtype=np.bool_) - ) + mask = self._reduction_null_mask(arr) + if mask is None: + mask = np.zeros(len(arr), dtype=np.bool_) if mask.all(): raise ValueError("argmin() called on a column where all values are null.") positions = np.where(~mask)[0] @@ -3385,9 +3413,9 @@ def argmax(self, axis=None, *, where=None): arr = np.asarray(self[:]) if arr.size == 0: raise ValueError("argmax() called on an empty column.") - mask = ( - self._null_mask_for(arr) if self.null_value is not None else np.zeros(len(arr), dtype=np.bool_) - ) + mask = self._reduction_null_mask(arr) + if mask is None: + mask = np.zeros(len(arr), dtype=np.bool_) if mask.all(): raise ValueError("argmax() called on a column where all values are null.") positions = np.where(~mask)[0] diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index dc8c99b1c..9cc049f08 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -721,9 +721,17 @@ def _mask_pred(self): sidecar" -- the latter meaning never-null, which the expression layer already handles by skipping the operand. - Fixed-shape ndarray columns are excluded for now: their values array - is N-D while the sidecar is one flag per row, so the two do not - broadcast against each other in a lazy expression. + **Fixed-shape ndarray columns are excluded**, and not merely because a + row-level flag is a different shape from an ``(n, *item_shape)`` values + array. Reshaping the sidecar to ``(n, 1, ...)`` looks like it should + fix that, and does not: ``blosc2.where`` returns the *predicate's* + shape rather than broadcasting, so the item dimension is silently + dropped from the values, and combining an ``(n, 1)`` predicate with the + ``(n,)`` row mask that :class:`NullableExpr` reductions use explodes + into ``(n, n)``. Row-level null propagation for ndarray columns needs + broadcasting support in the lazy layer; until then those columns get + their null handling from the NumPy-based reduction paths + (``Column._reduction_null_mask``), which are row-level throughout. """ if self.kind != NULL_MASK or self._col.is_ndarray: return None diff --git a/tests/ctable/test_null_mask_expressions.py b/tests/ctable/test_null_mask_expressions.py new file mode 100644 index 000000000..07b9c9ca6 --- /dev/null +++ b/tests/ctable/test_null_mask_expressions.py @@ -0,0 +1,327 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Expressions and reductions over mask-storage columns (Phase 5). + +The expression layer was already storage-agnostic — it consumes an opaque +boolean null predicate — so most of this is a differential check that mask and +sentinel columns agree wherever they are supposed to, and diverge only where +decision 6 says they must (NaN is a value under a mask, a null under a +sentinel). + +The reductions needed real work: ``argmin``/``argmax`` and the ndarray +reduction path both keyed off ``null_value is not None``, so under mask storage +they silently reduced over the *fill*. For an int column the fill is ``0``, +which is a plausible-looking minimum — the kind of wrong answer nobody notices. + +Two things this file pins as **deliberately not done**, because measurement +showed the plan's premises for them were wrong: + +* the summary-index ``min``/``max`` shortcut stays disabled for mask columns; +* ndarray columns still get no null predicate in the lazy layer. + +Both are explained where they are asserted. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 + + +def annotation_for(spec): + if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)): + return object + return spec.python_type + + +def one_col(spec, values, capacity=32): + Row = dataclasses.make_dataclass("R", [("v", annotation_for(spec), blosc2.field(spec))]) + t = blosc2.CTable(Row, expected_size=capacity) + t.extend([(v,) for v in values]) + return t + + +def mask_col(values, factory=blosc2.int64, **kw): + return one_col(factory(null_storage="mask", **kw), values) + + +# --------------------------------------------------------------------------- +# Reductions skip nulls rather than reducing over the fill +# --------------------------------------------------------------------------- + + +def test_min_ignores_the_zero_fill(): + """The int fill is 0 — a plausible-looking minimum for positive data.""" + assert mask_col([5, None, 1, 9])["v"].min() == 1 + + +def test_max_ignores_the_zero_fill(): + """...and a plausible-looking maximum for negative data.""" + assert mask_col([-5, None, -1])["v"].max() == -1 + + +def test_argmin_points_at_a_real_row(): + assert mask_col([5, None, 1, 9])["v"].argmin() == 2 + + +def test_argmax_points_at_a_real_row(): + assert mask_col([-5, None, -1])["v"].argmax() == 2 + + +def test_argmin_would_have_picked_the_fill(): + """Regression: the fill sits at row 1 and is smaller than every real value.""" + col = mask_col([5, None, 1, 9])["v"] + assert col[:][1] == 0 # the fill really is there in the values + assert col.argmin() == 2 # ...and is not what argmin reports + + +@pytest.mark.parametrize("op", ["argmin", "argmax"]) +def test_arg_reductions_raise_on_an_all_null_column(op): + col = mask_col([None, None])["v"] + with pytest.raises(ValueError, match="all values are null"): + getattr(col, op)() + + +def test_sum_and_mean_skip_nulls(): + col = mask_col([2.0, None, 4.0], factory=blosc2.float64)["v"] + assert col.sum() == 6.0 + assert col.mean() == 3.0 + + +def test_std_skips_nulls(): + col = mask_col([1.0, None, 3.0], factory=blosc2.float64)["v"] + assert col.std() == pytest.approx(1.0) + + +def test_reductions_ignore_capacity_padding(): + """Padding is fill-valued too, and must not reach a reduction either.""" + col = one_col(blosc2.int64(null_storage="mask"), [5, None, 1], capacity=64)["v"] + assert col.min() == 1 + assert col.max() == 5 + assert col.sum() == 6 + + +# --------------------------------------------------------------------------- +# ndarray columns: the gain is in the NumPy reduction paths +# --------------------------------------------------------------------------- + + +def ndarray_col(items, dtype=blosc2.int64): + spec = blosc2.ndarray((3,), dtype=dtype(), null_storage="mask") + return one_col(spec, items)["v"] + + +def test_ndarray_min_skips_null_rows(): + col = ndarray_col([np.array([1, 2, 3]), None, np.array([7, 8, 9])]) + assert col.min() == 1 # not 0, the fill item + + +def test_ndarray_max_skips_null_rows(): + col = ndarray_col([np.array([-1, -2, -3]), None, np.array([-7, -8, -9])]) + assert col.max() == -1 # not 0, the fill item + + +def test_ndarray_sum_skips_null_rows(): + col = ndarray_col([np.array([1, 2, 3]), None, np.array([7, 8, 9])]) + assert col.sum() == 30 + + +def test_ndarray_mean_divides_by_live_non_null_elements(): + col = ndarray_col([np.array([1, 2, 3]), None, np.array([7, 8, 9])]) + assert col.mean() == pytest.approx(30 / 6) + + +def test_ndarray_reduction_where_composes_with_nulls(): + spec = blosc2.ndarray((2,), dtype=blosc2.int64(), null_storage="mask") + Row = dataclasses.make_dataclass( + "R", + [ + ("e", object, blosc2.field(spec)), + ("k", int, blosc2.field(blosc2.int64())), + ], + ) + t = blosc2.CTable(Row, expected_size=16) + t.extend([(np.array([1, 1]), 0), (None, 1), (np.array([5, 5]), 1)]) + assert t["e"].sum(where="k == 1") == 10 + + +# --------------------------------------------------------------------------- +# Propagation through arithmetic and comparisons +# --------------------------------------------------------------------------- + + +def test_arithmetic_marks_null_rows_nan(): + expr = mask_col([1, None, 3])["v"] * 2 + values = np.asarray(expr[:3]) + assert values[0] == 2.0 + assert np.isnan(values[1]) + assert values[2] == 6.0 + + +def test_arithmetic_result_reduces_without_nan_poisoning(): + assert (mask_col([1, None, 3])["v"] * 2).sum() == 8.0 + + +def test_comparison_gives_null_rows_false(): + """SQL WHERE semantics: a null satisfies no comparison.""" + col = mask_col([1, None, 3])["v"] + assert (col > 0)[:3].tolist() == [True, False, True] + + +def test_where_over_a_mask_column_drops_nulls(): + t = mask_col([1, None, 3]) + assert t.where(t["v"] > 0)["v"][:].tolist() == [1, 3] + + +def test_not_equal_does_not_leak_nulls(): + """IEEE says nan != x is True; SQL says a null satisfies nothing.""" + col = mask_col([1, None, 3])["v"] + assert (col != 1)[:3].tolist() == [False, False, True] + + +def test_two_mask_columns_combine_their_nulls(): + Row = dataclasses.make_dataclass( + "R", + [ + ("a", int, blosc2.field(blosc2.int64(null_storage="mask"))), + ("b", int, blosc2.field(blosc2.int64(null_storage="mask"))), + ], + ) + t = blosc2.CTable(Row, expected_size=16) + t.extend([(1, 10), (None, 20), (3, None), (4, 40)]) + values = np.asarray((t["a"] + t["b"])[:4]) + assert values[0] == 11 + assert np.isnan(values[1]) + assert np.isnan(values[2]) + assert values[3] == 44 + + +def test_a_null_free_mask_column_costs_no_operand(): + """No sidecar means no predicate, so arithmetic stays a plain LazyExpr.""" + col = mask_col([1, 2, 3])["v"] + assert col._raw_null_pred() is None + assert isinstance(col * 2, blosc2.LazyExpr) + + +# --------------------------------------------------------------------------- +# Differential: mask and sentinel agree, except where decision 6 says not to +# --------------------------------------------------------------------------- + +AGREEING_CASES = [ + ("min", lambda c: c.min()), + ("max", lambda c: c.max()), + ("sum", lambda c: c.sum()), + ("mean", lambda c: c.mean()), + ("argmin", lambda c: c.argmin()), + ("argmax", lambda c: c.argmax()), + ("null_count", lambda c: c.null_count()), + ("is_null", lambda c: c.is_null().tolist()), + ("unique", lambda c: c.unique().tolist()), + ("gt", lambda c: (c > 2)[:4].tolist()), +] + + +@pytest.mark.parametrize(("label", "op"), AGREEING_CASES) +def test_mask_and_sentinel_agree_on_integer_data(label, op): + """Same logical data, two storages, one answer.""" + values = [5, None, 1, 9] + masked = mask_col(values)["v"] + sentinel = one_col(blosc2.int64(null_value=-999), [-999 if v is None else v for v in values])["v"] + assert op(masked) == op(sentinel) + + +def test_mask_and_sentinel_diverge_on_nan_by_design(): + """decision 6: NaN is a value under a mask, the null itself under a sentinel.""" + values = [1.0, float("nan"), 3.0] + masked = one_col(blosc2.float64(null_storage="mask"), values)["v"] + sentinel = one_col(blosc2.float64(null_value=float("nan")), values)["v"] + + assert masked.null_count() == 0 + assert sentinel.null_count() == 1 + # The NaN is data for the mask column, so it poisons the reduction the way + # NumPy does; for the sentinel column it is the null and is skipped. + assert np.isnan(masked.sum()) + assert sentinel.sum() == 4.0 + + +# --------------------------------------------------------------------------- +# Nullable bool needs no predicate rewrite under mask storage +# --------------------------------------------------------------------------- + + +def test_mask_bool_is_not_treated_as_a_sentinel_bool(): + col = mask_col([True, None, False], factory=blosc2.bool)["v"] + assert col._is_nullable_bool is False + + +def test_sentinel_bool_is_still_treated_as_one(): + col = one_col(blosc2.bool(nullable=True, null_value=255), [True, 255, False])["v"] + assert col._is_nullable_bool is True + + +def test_mask_bool_filters_directly(): + t = mask_col([True, None, False, True], factory=blosc2.bool) + assert t.where(t["v"])["v"][:].tolist() == [True, True] + + +# --------------------------------------------------------------------------- +# Deliberately not done — with the measurement that says why +# --------------------------------------------------------------------------- + + +def test_summary_minmax_shortcut_stays_disabled_for_mask_columns(tmp_path): + """Enabling it would make min() answer differently depending on the index. + + A mask float column's fill is NaN, which the summary builder drops — but so + is a *genuine* NaN, which decision 6 makes a value. The scan therefore + poisons to NaN while the summaries would report a real extremum. + """ + spec = blosc2.float64(null_storage="mask") + Row = dataclasses.make_dataclass("R", [("v", float, blosc2.field(spec))]) + path = str(tmp_path / "s.b2d") + t = blosc2.CTable(Row, expected_size=5, urlpath=path, mode="w") + t.extend([(1.0,), (float("nan"),), (5.0,), (None,), (3.0,)]) + t.close() + + reopened = blosc2.CTable.open(path, mode="a") + try: + assert reopened["v"]._summary_minmax_source() is None + # The scanned answer, which the shortcut would have contradicted. + assert np.isnan(reopened["v"].min()) + finally: + reopened.close() + + +def test_sentinel_nan_float_keeps_its_summary_shortcut(tmp_path): + """The contrast: there NaN *is* the null, so dropping it is exactly right.""" + spec = blosc2.float64(null_value=float("nan")) + Row = dataclasses.make_dataclass("R", [("v", float, blosc2.field(spec))]) + path = str(tmp_path / "n.b2d") + t = blosc2.CTable(Row, expected_size=4, urlpath=path, mode="w") + t.extend([(1.0,), (float("nan"),), (5.0,), (3.0,)]) + t.close() + + reopened = blosc2.CTable.open(path, mode="a") + try: + assert reopened["v"]._summary_minmax_source() is not None + assert reopened["v"].min() == 1.0 + finally: + reopened.close() + + +def test_ndarray_columns_still_get_no_lazy_null_predicate(): + """Not an oversight: see NullChannel._mask_pred for the two blockers.""" + col = ndarray_col([np.array([1, 2, 3]), None]) + assert col._raw_null_pred() is None + assert col._nulls.valid_pred() is None + # ...and the reduction paths, which are row-level NumPy, cover it instead. + assert col.min() == 1 From 3660f35a505f6ac23af963650c5770e7618c81b8 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 12:15:29 +0200 Subject: [PATCH 07/47] Round-trip Arrow and Parquet losslessly (mask-based-nulls 6) This is what the design is for. A sentinel steals a value from the dtype's range, so importing Arrow data has always been lossy for any column whose data happens to use that value -- and silently so, since nothing raises. Measured, and now pinned side by side with the mask result: pa.array([-128, None, 127], int8) -> [None, None, 127] pa.array(["", "__BLOSC2_NULL__", None]) -> ["", None, None] Under mask storage both come back exactly as they went in, as do nullable bool without the 255 reservation, full-range uint8, float64 carrying nan/inf/-0.0 as values, timestamps holding int64.min as a value, and fixed-width string/bytes fully occupying their declared width. Ships opt-in: null_storage="mask" on from_arrow/from_parquet, or a NullPolicy. The default stays "sentinel" until phase 9. The import error for a type with no available sentinel is not deleted -- it still fires for sentinel storage, which still cannot represent those types -- but it now names the way out. Two departures from the plan: * No np.packbits. arrow_slice already had the better answer for its sentinel path: hand the booleans to pyarrow and take the resulting array's data buffer. Arrow packs booleans and validity bitmaps identically, so this borrows pyarrow's own packing and makes the bit order unrepresentable-as- wrong rather than merely tested. The literal-bytes test was still worth writing and pins 0xDD, not the MSB-first 0xBB. * The round-trip contract cannot be spelled `.equals()`. pyarrow compares floats with IEEE semantics, so two identical arrays containing NaN compare unequal. The tests compare what is observable instead: the validity bitmap and the values under valid rows, NaN equal to NaN, signed zeros distinct. Values under valid=False are deliberately not compared -- the fill is not part of the format contract. Co-Authored-By: Claude Opus 5 --- plans/mask-based-nulls.md | 49 ++- src/blosc2/_utf8_array.py | 43 ++- src/blosc2/ctable.py | 195 ++++++++---- src/blosc2/ctable_nulls.py | 37 +++ tests/ctable/test_null_mask_arrow.py | 432 +++++++++++++++++++++++++++ 5 files changed, 682 insertions(+), 74 deletions(-) create mode 100644 tests/ctable/test_null_mask_arrow.py diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 39b9ad13a..aad188ae4 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,13 +1,14 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS — Phases 0–5 landed 2026-08-08.** Mask columns are fully usable in memory -> and on disk, and correct through expressions and reductions; next up is Phase 6 (Arrow/Parquet), -> which is what the whole design is *for*. Four premises were disproven during implementation and +> **Status: IN PROGRESS — Phases 0–6 landed 2026-08-08.** Lossless Arrow/Parquet round-trip now +> works for every V1 kind, **opt-in** via `null_storage="mask"`; next up is Phase 7 (sort + +> groupby), then Phase 8 (migration + docs). Six premises were disproven during implementation and > are corrected in place, each in a blockquote beside the text it corrects: the index path cannot > be fixed by a null-aware expression (§Expression layer), the bool dtype-flip cannot move out of > `__init__` (§Schema layer), ndarray columns do not get lazy null propagation for free -> (§Expression layer), and the "free" summary min/max fast path is unsound (§Reductions). -> Drafted 2026-08-08. +> (§Expression layer), the "free" summary min/max fast path is unsound (§Reductions), `np.packbits` +> is not needed and avoiding it is safer (§Arrow/Parquet), and `.equals()` cannot express the +> round-trip contract (§Arrow/Parquet). Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -439,6 +440,15 @@ Arrow validity bitmaps are LSB-first. This is the easiest bug in the plan to int hardest to catch — a round-trip test passes with **either** bit order if import unpacks the same way. Pin it with a test asserting the literal packed bytes for a known pattern. +> **As built (2026-08-08): `np.packbits` is not needed at all, and avoiding it is strictly safer.** +> `arrow_slice` already had the answer for its sentinel path — `pa.array(~mask).buffers()[1]`. +> Arrow packs booleans and validity bitmaps identically (LSB-first), so handing pyarrow the +> booleans and taking the resulting array's *data* buffer borrows its packing and makes the bit +> order unrepresentable-as-wrong rather than merely tested. The mask path does the same with +> `pa.array(valid).buffers()[1]`. The literal-bytes test was still worth writing and is the useful +> half of the advice above: `test_utf8_validity_bitmap_is_lsb_first` pins +> `[valid, null, valid, valid, valid, null, valid, valid]` to `0xDD`, not the MSB-first `0xBB`. + **Import** — in `_compiled_columns_from_arrow` (`:7369-7482`), when the resolved storage is `"mask"` the entire sentinel-selection block is skipped and **the `"no null_value sentinel is available"` error at `:7457` never fires**. That single deletion is what makes nullable bool, @@ -461,6 +471,23 @@ nullable `bool`; `int8`/`uint8` using **all 256 values** plus nulls; `float64` c separate nulls; `string(max_length=4)`/`bytes(max_length=4)` fully occupying the width. **None of these round-trip under sentinels.** Same list for Parquet. +> **Correction (2026-08-08): `.equals()` cannot express this contract.** `pyarrow.Array.equals` +> compares floats with IEEE semantics, so two *identical* arrays containing NaN compare unequal — +> the float case of the list above fails by construction, whatever the implementation does. The +> tests use `assert_same_logical` instead, which compares what is actually observable: the validity +> bitmap, and the values under valid rows, with NaN equal to NaN and signed zeros kept distinct. +> Values under `valid=False` are deliberately **not** compared — the fill is explicitly not part of +> the format contract (decision 5), so asserting on it would pin something the design says may +> change. +> +> **The "none of these round-trip under sentinels" claim is understated, and now measured.** The +> sentinel path does not fail — it *silently returns different data*. `pa.array([-128, None, 127], +> int8)` imports and re-exports as `[None, None, 127]`, because `-128` is the sentinel `int8` +> picks; `["", "__BLOSC2_NULL__", None]` comes back as `["", None, None]`, because that literal is +> the utf8 sentinel. Both are pinned side by side against the mask result in +> `test_sentinel_storage_is_lossy_where_mask_storage_is_not`, which is the single most direct +> statement of why this project exists. + ### Reductions and summary indexes Mechanical: `_ndarray_values_for_reduction` (`:2843-2867`) and `argmin`/`argmax` (`:3218-3271`) @@ -676,6 +703,16 @@ file, round-trip it, and assert `pq.read_table(out).equals(pq.read_table(in))` impossible today. Also re-run the OFF importer round-trip (`plans/ctable-nulls.md` §Tests) and confirm the `nullable_scalar_wrapped_as_singleton_list` workaround can be deleted. +> **Correction (2026-08-08).** The smoke test is now a real test rather than a manual one — +> `test_parquet_round_trip_is_lossless`, parametrized over the whole contract list. +> +> The `nullable_scalar_wrapped_as_singleton_list` workaround **cannot be deleted, and does not need +> to be.** `src/blosc2/cli/parquet_to_blosc2.py` stopped *producing* it before this work began — the +> conversion table at `:410-496` emits `nullable_scalar_sentinel` for that case now. The two +> surviving references (`:1405-1406`) are in the *export* path, which reads the tag out of archive +> metadata to unwrap what an older version wrote. That is a permanent backward-compatibility reader, +> in the same category as `_is_nullable_bool` and its rewrite sites: it stays forever. + ## Phasing Each phase is independently landable. **The default does not flip until Phase 9**, which is a @@ -690,7 +727,7 @@ default-created tables require them. | 3 | ✅ **Storage sidecar.** 5 methods × 4 backends, lazy creation (absent key = all valid); `_grow`/`trim_capacity`/`compact`/`_save_to_storage`/`to_cframe`/`load`, **plus `copy()`'s in-memory path**, which the section above had missed; companion-suffix loop in delete/rename. `tests/ctable/test_null_persistence.py` (38 tests) drives it with a hand-built mask. | M | Low | | 4 | ✅ **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`; **plus every gather-and-rebuild path** (`sort_by` ×3, `take`, `slice`), which this section had not listed. Mask columns fully usable. `tests/ctable/test_null_mask_api.py` (90 tests). | **L** | **High** (turned out to be the reference cycle, not `__setitem__`) | | 5 | ✅ **Expressions + reductions.** `_ndarray_values_for_reduction`, argmin/argmax (both were reducing over the *fill*), `_reduction_null_mask` as the one storage-agnostic entry point. `_raw_null_pred`/`_lazy_nonnull_mask`/`_is_nullable_bool` needed nothing — Phases 0–4 had already made them storage-agnostic. **The ndarray-propagation gain is not real and was not done**, and the free summary fast path is unsound; both corrections are above. `tests/ctable/test_null_mask_expressions.py` (39 tests). | S (was M) | Low (was Med) | -| 6 | **Arrow/Parquet.** Import + export for all V1 kinds, `packbits`/`unpackbits` LSB-first, `arrow_slice(validity=)`, delete the "no sentinel available" import error. Ships **opt-in** (`null_storage="mask"`); the default stays `"sentinel"`. | M | Med | +| 6 | ✅ **Arrow/Parquet.** Import + export for all V1 kinds; `arrow_slice(valid=)`; `null_storage=` on `from_arrow`/`from_parquet`; the "no sentinel available" import error now names the way out instead of being deleted (it still fires for sentinel storage, which still cannot represent those types). No `packbits` — pyarrow's own packing is borrowed instead. Ships **opt-in**; the default stays `"sentinel"`. `tests/ctable/test_null_mask_arrow.py` (42 tests). | M | Med | | 7 | **Sort + groupby.** `_build_lex_keys`, `_sorted_positions_from_full_index` (big I/O win), `_utf8_rank_arrays(valid=)`, groupby `_null_mask` threading. | M | Med-High | | 8 | **Migration + docs.** `convert_nulls`, `Column.null_storage`, `info()`, `doc/reference/ctable.rst` null-policy rewrite, release notes. | S–M | Low | | 9 | **Default flips to `"mask"`.** A one-line `NullPolicy` change plus release notes — lossless round-trip is why the default exists. Lands **no earlier than one release after Phase 6** so older readers in the wild already understand schema version 3. | S | Low | diff --git a/src/blosc2/_utf8_array.py b/src/blosc2/_utf8_array.py index e7ad86182..4dabac4ce 100644 --- a/src/blosc2/_utf8_array.py +++ b/src/blosc2/_utf8_array.py @@ -452,8 +452,16 @@ def _bytes_used(self) -> int: return self._bytes_used_cache def _coerce(self, value: Any) -> str: - """Coerce *value* to ``str``, mapping ``None`` to the null sentinel.""" + """Coerce *value* to ``str``, mapping ``None`` to whatever fills a null slot. + + Under sentinel storage that is the sentinel string itself. Under mask + storage it is the zero-length fill, and what actually records the null + is the column's validity sidecar — written by the caller alongside this + value, not here. + """ if value is None: + if getattr(self._spec, "uses_mask", False): + return "" null_value = getattr(self._spec, "null_value", None) if null_value is None: raise TypeError("Column of utf8 strings is not nullable; received None.") @@ -958,14 +966,25 @@ def order_masks_span(self, value: str, a: int, b: int) -> tuple[np.ndarray, np.n gt[rows] = row_gt return lt, gt - def arrow_slice(self, pa, a: int, b: int, null_value: str | None = None): + def arrow_slice(self, pa, a: int, b: int, null_value: str | None = None, *, valid=None): """Persisted rows ``[a, b)`` as a ``pyarrow.LargeStringArray``. The storage layout (int64 offsets + UTF-8 byte blob) is exactly Arrow's ``large_string`` layout, so the array is built directly from the raw buffers with no per-row decode or Python string objects. - When *null_value* is given, rows equal to the sentinel become Arrow - nulls (matched on raw bytes, still without decoding). + + Nullity comes from whichever channel the column uses. Pass *valid* -- + a boolean array over rows ``[a, b)``, ``True`` where the row is not + null -- for mask storage; pass *null_value* for sentinel storage, where + rows equal to the sentinel become Arrow nulls, matched on raw bytes and + still without decoding. + + The validity bitmap is produced by handing the booleans to pyarrow and + taking the resulting array's data buffer, rather than packing the bits + here. Arrow packs booleans and validity bitmaps the same way (LSB + first), so this borrows pyarrow's own packing and removes any chance of + emitting the bitmap in the wrong bit order -- a bug a round-trip test + cannot catch, because import would unpack it the same wrong way. """ n = b - a offs = np.ascontiguousarray(self._offsets[a : b + 1], dtype=np.int64) @@ -974,11 +993,17 @@ def arrow_slice(self, pa, a: int, b: int, null_value: str | None = None): data = np.ascontiguousarray(self._data[start:end]) if end > start else np.empty(0, dtype=np.uint8) validity = None null_count = 0 - if null_value is not None and n > 0: - mask = self.equal_mask_span(null_value, a, b) - null_count = int(mask.sum()) - if null_count: - validity = pa.array(~mask).buffers()[1] + if n > 0: + if valid is not None: + valid = np.ascontiguousarray(valid, dtype=np.bool_) + null_count = n - int(np.count_nonzero(valid)) + if null_count: + validity = pa.array(valid).buffers()[1] + elif null_value is not None: + mask = self.equal_mask_span(null_value, a, b) + null_count = int(mask.sum()) + if null_count: + validity = pa.array(~mask).buffers()[1] return pa.LargeStringArray.from_buffers( n, pa.py_buffer(rel), pa.py_buffer(data), validity, null_count ) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 2ec6a8d40..6f6055e0f 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -38,6 +38,7 @@ NULL_MASK, NULL_SENTINEL, NullChannel, + fill_value_for, is_nan_sentinel, kind_of_spec, rewrite_null_predicates, @@ -7522,15 +7523,17 @@ def iter_arrow_batches( # noqa: C901 # Dense root table: logical rows == persisted rows, so # export straight from the offsets/bytes buffers with # no per-row decode (storage is already Arrow layout). - arrays.append(arr8.arrow_slice(pa, start, stop, nv)) + arrays.append( + arr8.arrow_slice(pa, start, stop, nv, valid=col._nulls.valid_slice(start, stop)) + ) continue - values = col[start:stop] # StringDType array with sentinel nulls - null_mask = col._null_mask_for(values) if nv is not None else None + values = col[start:stop] # StringDType array; nulls per this column's channel + null_mask = col._nulls.null_mask_slice(values, start, stop) arrays.append( pa.array( values.astype(object), type=self._pa_type_from_spec(pa, spec), - mask=null_mask if null_mask is not None and null_mask.any() else None, + mask=null_mask, ) ) continue @@ -7569,7 +7572,11 @@ def iter_arrow_batches( # noqa: C901 if col.is_ndarray: spec = self._schema.columns_by_name[name].spec values = np.asarray(col[start:stop]) - null_mask = col._null_mask_for(values) if col.null_value is not None else None + # Row-level under mask storage. A sentinel ndarray column + # keeps the older, lossier rule -- a row is null only when + # *every* element equals the sentinel -- because that is the + # only thing its storage can express. + null_mask = col._nulls.null_mask_slice(values, start, stop) pa_type = self._pa_type_from_spec(pa, spec) flat_values = np.ascontiguousarray(values.reshape(-1)) pa_values = pa.array(flat_values, type=pa_type.value_type) @@ -7577,33 +7584,28 @@ def iter_arrow_batches( # noqa: C901 pa.FixedSizeListArray.from_arrays( pa_values, type=pa_type, - mask=( - pa.array(null_mask, type=pa.bool_()) - if null_mask is not None and null_mask.any() - else None - ), + mask=(pa.array(null_mask, type=pa.bool_()) if null_mask is not None else None), ) ) continue arr = np.asarray(col[start:stop]) - nv = col.null_value - null_mask = col._null_mask_for(arr) if nv is not None else None - has_nulls = null_mask is not None and bool(null_mask.any()) - if arr.dtype.kind == "U": - values = arr.tolist() - if has_nulls: - values = [None if null_mask[i] else v for i, v in enumerate(values)] - arrays.append(pa.array(values, type=pa.string())) - elif arr.dtype.kind == "S": - values = arr.tolist() - if has_nulls: - values = [None if null_mask[i] else v for i, v in enumerate(values)] - arrays.append(pa.array(values, type=pa.large_binary())) + null_mask = col._nulls.null_mask_slice(arr, start, stop) + if arr.dtype.kind in "US": + # pyarrow reads the mask alongside the values, so the null + # slots need no substitution here — under mask storage they + # already hold the fill, and under a sentinel the mask says + # to ignore whatever is there. + pa_type = pa.string() if arr.dtype.kind == "U" else pa.large_binary() + arrays.append(pa.array(arr.tolist(), type=pa_type, mask=null_mask)) elif ( self._schema.columns_by_name.get(name) is not None and self._schema.columns_by_name[name].spec.to_metadata_dict().get("kind") == "bool" ): - arrays.append(pa.array(arr == 1, mask=null_mask if has_nulls else None, type=pa.bool_())) + # A sentinel bool is physically uint8 (0/1/255) and needs the + # compare; a mask bool is already np.bool_ and must not get + # one, since `arr == 1` would be a no-op at best. + values = arr == 1 if col._is_nullable_bool else arr + arrays.append(pa.array(values, mask=null_mask, type=pa.bool_())) elif self._schema.columns_by_name.get(name) is not None and isinstance( self._schema.columns_by_name[name].spec, timestamp ): @@ -7612,12 +7614,12 @@ def iter_arrow_batches( # noqa: C901 arrays.append( pa.array( values, - mask=null_mask if has_nulls else None, + mask=null_mask, type=pa.timestamp(spec.unit, tz=spec.timezone), ) ) else: - arrays.append(pa.array(arr, mask=null_mask if has_nulls else None)) + arrays.append(pa.array(arr, mask=null_mask)) yield pa.RecordBatch.from_arrays(arrays, names=arrow_names) def to_arrow(self): @@ -7688,6 +7690,7 @@ def _arrow_type_to_spec( # noqa: C901 field_metadata=None, string_max_length=None, null_value=None, + null_storage=None, nullable=False, object_fallback: bool = False, ): @@ -7716,7 +7719,13 @@ def _arrow_type_to_spec( # noqa: C901 f"Arrow fixed-size-list metadata shape {shape} has size {int(np.prod(shape))}, " f"but the Arrow list size is {pa_type.list_size}." ) - return b2s.ndarray(shape, dtype=value_dtype, nullable=nullable, null_value=null_value) + return b2s.ndarray( + shape, + dtype=value_dtype, + nullable=nullable, + null_value=null_value, + null_storage=null_storage, + ) if pa.types.is_dictionary(pa_type): vt = pa_type.value_type @@ -7773,13 +7782,19 @@ def _arrow_type_to_spec( # noqa: C901 ] if pa.types.is_timestamp(pa_type): return b2s.timestamp( - unit=pa_type.unit, timezone=pa_type.tz, nullable=nullable, null_value=null_value + unit=pa_type.unit, + timezone=pa_type.tz, + nullable=nullable, + null_value=null_value, + null_storage=null_storage, ) for arrow_t, spec_cls in mapping: if pa_type == arrow_t: if null_value is not None and getattr(spec_cls, "supports_sentinel", False): return spec_cls(null_value=null_value) + if null_storage is not None: + return spec_cls(null_storage=null_storage) return spec_cls() if pa.types.is_list(pa_type) or pa.types.is_large_list(pa_type): @@ -7835,16 +7850,16 @@ def _arrow_type_to_spec( # noqa: C901 return b2s.vlstring(nullable=nullable) # No fixed-width threshold given: store as a variable-length # utf8 column (offsets + bytes, StringDType reads). - return b2s.utf8(nullable=nullable, null_value=null_value) + return b2s.utf8(nullable=nullable, null_value=null_value, null_storage=null_storage) max_length = max(string_max_length, len(null_value) if null_value is not None else 1, 1) - return b2s.string(max_length=max_length, null_value=null_value) + return b2s.string(max_length=max_length, null_value=null_value, null_storage=null_storage) if _is_arrow_binary_type(pa, pa_type): if string_max_length is None: # No fixed-width threshold given: store as variable-length scalar bytes. return b2s.vlbytes(nullable=nullable) max_length = max(string_max_length, len(null_value) if null_value is not None else 1, 1) - return b2s.bytes(max_length=max_length, null_value=null_value) + return b2s.bytes(max_length=max_length, null_value=null_value, null_storage=null_storage) if object_fallback: return b2s.object(nullable=nullable) @@ -7872,8 +7887,12 @@ def _compiled_columns_from_arrow( *, auto_null_sentinels: bool, object_fallback: bool = False, + null_storage: str | None = None, ): null_policy = get_null_policy() + # Only inferred schemas consult the policy; extending an existing table + # never reaches here, so its stored null_storage always wins. + storage_pref = null_storage if null_storage is not None else null_policy.null_storage column_null_values = null_policy.column_null_values schema_names = set(schema.names) unknown_null_values = set(column_null_values) - schema_names @@ -7926,36 +7945,42 @@ def _compiled_columns_from_arrow( f"column_null_values is not supported for vlbytes/vlstring column {name!r}; " "these columns represent nulls as native None." ) + # Kinds that carry their own nullity (native None, a reserved code, + # a nested layout) and so never take either scalar null channel. + handles_own_nulls = ( + field_is_list + or field_is_struct + or field_is_dictionary + or field_is_varlen_scalar + or field_is_object_fallback + ) + # Mask storage needs no sentinel, so none of the selection below + # applies — and neither does its failure mode. This is what makes + # nullable bool, full-range int8/uint8 and free-text utf8 + # importable at all. + use_mask = ( + storage_pref == NULL_MASK + and field.nullable + and not handles_own_nulls + and not has_null_value_override + ) if has_null_value_override: null_value = column_null_values[name] - elif ( - auto_null_sentinels - and field.nullable - and not ( - field_is_list - or field_is_struct - or field_is_dictionary - or field_is_varlen_scalar - or field_is_object_fallback - ) - ): + elif not use_mask and auto_null_sentinels and field.nullable and not handles_own_nulls: arrow_type_for_null = field.type.value_type if field_is_ndarray else field.type null_value = cls._auto_null_sentinel(pa, arrow_type_for_null, null_policy=null_policy) if ( arrow_col is not None and arrow_col.null_count - and not ( - field_is_list - or field_is_struct - or field_is_dictionary - or field_is_varlen_scalar - or field_is_object_fallback - ) + and not handles_own_nulls and null_value is None + and not use_mask ): raise TypeError( - f"Column {name!r} contains Parquet nulls. Provide a CTable schema with a " - "null_value sentinel for this column." + f"Column {name!r} contains Parquet nulls, and no null_value sentinel is " + f"available for its type. Provide a CTable schema with a null_value sentinel " + f"for this column, or import with null_storage='mask' to keep nullity in a " + f"sidecar validity array (which every type supports)." ) spec = cls._arrow_type_to_spec( pa, @@ -7964,6 +7989,7 @@ def _compiled_columns_from_arrow( field_metadata=field.metadata, string_max_length=column_string_max_length, null_value=null_value, + null_storage=NULL_MASK if use_mask else None, nullable=field.nullable, object_fallback=object_fallback, ) @@ -8235,7 +8261,9 @@ def _write_arrow_batches(cls, obj, batches, columns, new_cols, new_valid) -> Non while end > len(new_valid): obj._grow() new_valid = obj._valid_rows - pos = cls._write_arrow_batch(batch, columns, new_cols, new_valid, pos, list_normalizers, writers) + pos = cls._write_arrow_batch( + obj, batch, columns, new_cols, new_valid, pos, list_normalizers, writers + ) for writer in writers.values(): writer.flush() # All imported rows are valid; mark them in a single aligned write. @@ -8254,7 +8282,7 @@ def _write_arrow_batches(cls, obj, batches, columns, new_cols, new_valid) -> Non @classmethod def _write_arrow_batch( - cls, batch, columns, new_cols, new_valid, pos: int, list_normalizers, writers + cls, obj, batch, columns, new_cols, new_valid, pos: int, list_normalizers, writers ) -> int: m = len(batch) if m == 0: @@ -8274,6 +8302,12 @@ def _write_arrow_batch( values = [normalizer(value) for value in values] new_cols[col.name].extend(values, validate=False) elif cls._is_varlen_scalar_column(col): + # utf8 is a varlen kind that can still use a sidecar; the other + # varlen kinds keep their native None cells and need nothing. + if getattr(col.spec, "uses_mask", False) and arrow_col.null_count: + obj._ensure_null_mask(col.name)[pos : pos + m] = arrow_col.is_valid().to_numpy( + zero_copy_only=False + ) new_cols[col.name].extend(arrow_col.to_pylist()) elif cls._is_dictionary_column(col): import pyarrow as _pa @@ -8284,6 +8318,19 @@ def _write_arrow_batch( else: # Plain string array: encode values into the dictionary. new_cols[col.name][pos : pos + m] = arrow_col.to_pylist() + elif getattr(col.spec, "uses_mask", False): + values, valid = cls._arrow_column_to_numpy_masked(arrow_col, col) + if valid is not None: + # A batch with no nulls writes nothing, so a nullable-but + # -null-free column still ends up with no sidecar at all. + # A fresh sidecar is created all-True, so the rows already + # written before this first null need no back-fill. + # + # No chunk-aligned writer here, unlike the values: the + # sidecar is one byte per row and compresses to almost + # nothing, so a straddling write is not worth avoiding. + obj._ensure_null_mask(col.name)[pos : pos + m] = valid + writers[col.name].append(values) else: writers[col.name].append(cls._arrow_column_to_numpy(arrow_col, col)) return pos + m @@ -8291,10 +8338,18 @@ def _write_arrow_batch( @staticmethod def _arrow_column_to_numpy(arrow_col, col: CompiledColumn) -> np.ndarray: nv = getattr(col.spec, "null_value", None) + uses_mask = getattr(col.spec, "uses_mask", False) + # What a null slot ends up holding in storage. Under a sentinel that + # value *is* the null marker; under mask storage it is an arbitrary, + # unobservable fill and the sidecar carries the nullity instead. + fill = fill_value_for(col.spec) if uses_mask else nv if col.spec.to_metadata_dict().get("kind") == "bool" and col.dtype == np.dtype(np.uint8): - return np.array([nv if v is None else int(v) for v in arrow_col.to_pylist()], dtype=np.uint8) + return np.array([fill if v is None else int(v) for v in arrow_col.to_pylist()], dtype=np.uint8) if isinstance(col.spec, NDArraySpec): values = arrow_col.to_pylist() + if uses_mask and arrow_col.null_count: + item = np.full(col.spec.item_shape, fill, dtype=col.spec.dtype) + values = [item if v is None else v for v in values] arr = CTable._coerce_ndarray_batch(col.name, col.spec, values, len(values)) return arr.reshape((len(values), *col.spec.item_shape)) if isinstance(col.spec, timestamp): @@ -8303,27 +8358,44 @@ def _arrow_column_to_numpy(arrow_col, col: CompiledColumn) -> np.ndarray: .astype(f"datetime64[{col.spec.unit}]") .astype(np.int64) ) + # NaT already decodes to int64.min, which is exactly the mask fill, + # so only a sentinel that differs from it needs remapping. if arrow_col.null_count and nv is not None and int(nv) != int(np.iinfo(np.int64).min): arr[arr == np.iinfo(np.int64).min] = int(nv) return arr.astype(col.dtype, copy=False) if col.dtype.kind in "US": values = arrow_col.to_pylist() - if nv is not None: - values = [nv if v is None else v for v in values] + if fill is not None: + values = [fill if v is None else v for v in values] max_len = col.spec.max_length too_long = [v for v in values if v is not None and len(v) > max_len] if too_long: raise ValueError(f"Column {col.name!r} contains values longer than max_length={max_len}.") return np.array(values, dtype=col.dtype) if arrow_col.null_count: - if nv is None: + if fill is None: raise TypeError( f"Column {col.name!r} contains Arrow/Parquet nulls. Provide a CTable schema " - "with a null_value sentinel for this column." + "with a null_value sentinel for this column, or import with " + "null_storage='mask'." ) - arrow_col = arrow_col.fill_null(nv) + arrow_col = arrow_col.fill_null(fill) return arrow_col.to_numpy(zero_copy_only=False).astype(col.dtype) + @staticmethod + def _arrow_column_to_numpy_masked(arrow_col, col: CompiledColumn): + """Split an Arrow column into ``(values, valid)`` for a mask column. + + The source's own validity is authoritative — no value is inspected to + decide what is null, which is precisely why a mask import can accept + types no sentinel could represent. *valid* is ``None`` when the batch + contained no nulls, so the caller writes no sidecar for it. + """ + values = CTable._arrow_column_to_numpy(arrow_col, col) + if not arrow_col.null_count: + return values, None + return values, arrow_col.is_valid().to_numpy(zero_copy_only=False) + @staticmethod def _arrow_schema_metadata(schema) -> dict[str, Any]: import base64 @@ -8502,6 +8574,7 @@ def from_arrow( # noqa: C901 capacity_hint: int | None = None, string_max_length: int | Mapping[str, int] | None = None, auto_null_sentinels: bool = True, + null_storage: Literal["mask", "sentinel"] | None = None, blosc2_batch_size: int | None = _BATCH_SIZE_DEFAULT, blosc2_items_per_block: int | None = None, list_serializer: Literal["msgpack", "arrow"] = "msgpack", @@ -8653,6 +8726,7 @@ def from_arrow( # noqa: C901 string_max_length, auto_null_sentinels=auto_null_sentinels, object_fallback=object_fallback, + null_storage=null_storage, ) cls._apply_arrow_column_cparams(columns, column_cparams) for col in columns: @@ -8772,6 +8846,7 @@ def from_parquet( # noqa: C901 dparams=None, validate: bool = False, auto_null_sentinels: bool = True, + null_storage: Literal["mask", "sentinel"] | None = None, blosc2_batch_size: int | None = _BATCH_SIZE_DEFAULT, blosc2_items_per_block: int | None = None, list_serializer: Literal["msgpack", "arrow"] = "arrow", @@ -9025,6 +9100,7 @@ def _limited_batches(batch_iter, limit: int): capacity_hint=max_rows, string_max_length=string_max_length, auto_null_sentinels=auto_null_sentinels, + null_storage=null_storage, blosc2_batch_size=blosc2_batch_size, blosc2_items_per_block=blosc2_items_per_block, list_serializer=list_serializer, @@ -9061,6 +9137,7 @@ def _limited_batches(batch_iter, limit: int): capacity_hint=_capacity_hint, string_max_length=string_max_length, auto_null_sentinels=auto_null_sentinels, + null_storage=null_storage, blosc2_batch_size=blosc2_batch_size, blosc2_items_per_block=blosc2_items_per_block, list_serializer=list_serializer, diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index 9cc049f08..c43f2684d 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -519,6 +519,43 @@ def null_mask(self) -> np.ndarray: return np.array([v is None for v in col], dtype=np.bool_) return self.mask_for_values(col[:]) + def valid_slice(self, start: int, stop: int): + """Physical validity for rows ``[start, stop)``, or ``None`` if all valid. + + Physical, not logical: this serves the export paths that read straight + from the storage buffers, which only run on a dense root table where + the two coincide. Use :meth:`null_mask_slice` everywhere else. + """ + valid = self.valid_array() + return None if valid is None else np.asarray(valid[start:stop]) + + def null_mask_slice(self, values, start: int, stop: int): + """Null flags for the logical rows ``[start, stop)``, or ``None``. + + ``None`` -- rather than an all-False array -- is the answer when + nothing in the range is null, because that is what pyarrow wants for + "this array needs no validity buffer". + + *values* is whatever ``col[start:stop]`` already returned, so the + sentinel kinds answer from it instead of reading the column a second + time; the mask kind ignores it and reads its sidecar at the same rows. + """ + kind = self.kind + col = self._col + if kind == NULL_MASK: + valid = self.valid_array() + if valid is None: + return None + if col._has_identity_positions(): + null = ~np.asarray(valid[start:stop]) + else: + null = ~np.asarray(valid[col._resolve_live_positions()[start:stop]]) + elif kind == NULL_SENTINEL: + null = sentinel_mask(values, self.sentinel, item_ndim=col.item_ndim if col.is_ndarray else 0) + else: + return None + return null if null.any() else None + def is_null_at(self, index: int) -> builtin_bool: """Whether the value at *logical* row *index* is null. diff --git a/tests/ctable/test_null_mask_arrow.py b/tests/ctable/test_null_mask_arrow.py new file mode 100644 index 000000000..32d59172f --- /dev/null +++ b/tests/ctable/test_null_mask_arrow.py @@ -0,0 +1,432 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Lossless Arrow/Parquet interop for mask-storage columns (Phase 6). + +This is what the whole design is for. A sentinel steals a value from the +dtype's range, so importing Arrow data has always been lossy for any column +whose data happens to use that value — silently lossy, since nothing raises. +``test_sentinel_storage_is_lossy_where_mask_storage_is_not`` measures exactly +that, and is the reason the rest of this file exists. + +Phase 6 ships **opt-in**: ``null_storage="mask"`` on ``from_arrow`` / +``from_parquet``, or a ``NullPolicy``. The default stays ``"sentinel"`` until +Phase 9, so every existing caller is unaffected. + +On comparing round-trips: ``pyarrow.Array.equals`` uses IEEE semantics, so two +*identical* arrays containing NaN compare unequal, and the plan's literal +``to_arrow(from_arrow(x)).equals(x)`` contract is unachievable for float data +by construction. :func:`assert_same_logical` compares what is actually +observable — the validity bitmap, and the values under valid rows — treating +NaN as equal to NaN and keeping signed zeros distinct. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +import blosc2 + +pa = pytest.importorskip("pyarrow") +pq = pytest.importorskip("pyarrow.parquet") + + +# --------------------------------------------------------------------------- +# Comparison helper +# --------------------------------------------------------------------------- + + +def _flat(column): + """A plain Array, whether the input was one already or a ChunkedArray.""" + return column.combine_chunks() if hasattr(column, "combine_chunks") else column + + +def assert_same_logical(got, want): + """Assert two Arrow columns hold the same observable data. + + Values sitting under ``valid=False`` are deliberately *not* compared: they + are the fill, which is explicitly not part of the format contract. + """ + got, want = _flat(got), _flat(want) + assert len(got) == len(want), "length" + assert got.null_count == want.null_count, "null_count" + + got_valid = got.is_valid().to_numpy(zero_copy_only=False) + want_valid = want.is_valid().to_numpy(zero_copy_only=False) + assert np.array_equal(got_valid, want_valid), "validity bitmap" + + for i in range(len(got)): + if not want_valid[i]: + continue + g, w = got[i], want[i] + if pa.types.is_timestamp(g.type): + g, w = g.cast(pa.int64()), w.cast(pa.int64()) + g, w = g.as_py(), w.as_py() + if isinstance(g, float) and isinstance(w, float): + assert g == w or (np.isnan(g) and np.isnan(w)), f"row {i}: {g!r} != {w!r}" + if g == 0.0: + assert np.copysign(1, g) == np.copysign(1, w), f"row {i}: signed zero" + else: + assert g == w, f"row {i}: {g!r} != {w!r}" + + +def round_trip(arrow_array, **kwargs): + """Import one Arrow column into a mask-backed CTable and export it again.""" + table = pa.table({"v": arrow_array}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask", **kwargs) + return ct, ct.to_arrow().column("v") + + +# --------------------------------------------------------------------------- +# The round-trip contract: none of these survive under a sentinel +# --------------------------------------------------------------------------- + +ROUND_TRIP_CASES = [ + ("bool", pa.array([True, None, False, True], type=pa.bool_())), + ("int8_full_range", pa.array([*range(-128, 128), None], type=pa.int8())), + ("uint8_full_range", pa.array([*range(256), None], type=pa.uint8())), + ( + "float64_specials", + pa.array( + [float("nan"), None, 0.0, -0.0, float("inf"), float("-inf")], + type=pa.float64(), + ), + ), + ( + "utf8_free_text", + pa.array(["", "\x00", "__BLOSC2_NULL__", None, "\U0001f389x"], type=pa.string()), + ), + ( + "timestamp_int64_min", + pa.array( + [ + np.datetime64("2020-01-01", "us"), + None, + np.datetime64(np.iinfo(np.int64).min + 1, "us"), + ], + type=pa.timestamp("us"), + ), + ), +] + + +@pytest.mark.parametrize(("label", "arrow_array"), ROUND_TRIP_CASES) +def test_arrow_round_trip_is_lossless(label, arrow_array): + _, exported = round_trip(arrow_array) + assert_same_logical(exported, arrow_array) + + +@pytest.mark.parametrize(("label", "arrow_array"), ROUND_TRIP_CASES) +def test_parquet_round_trip_is_lossless(label, arrow_array, tmp_path): + src = str(tmp_path / "in.parquet") + out = str(tmp_path / "out.parquet") + original = pa.table({"v": arrow_array}) + pq.write_table(original, src) + + ct = blosc2.CTable.from_parquet(src, null_storage="mask") + ct.to_parquet(out) + assert_same_logical(pq.read_table(out).column("v"), original.column("v")) + + +@pytest.mark.parametrize("max_length", [4]) +def test_fixed_width_string_fully_occupying_its_width(max_length): + arrow_array = pa.array(["abcd", None, "", "wxyz"], type=pa.string()) + table = pa.table({"v": arrow_array}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask", string_max_length=max_length) + assert ct["v"].dtype == np.dtype(f"U{max_length}") # no widening for a sentinel + assert_same_logical(ct.to_arrow().column("v"), arrow_array) + + +def test_fixed_width_bytes_fully_occupying_its_width(): + arrow_array = pa.array([b"abcd", None, b"", b"wxyz"], type=pa.binary()) + table = pa.table({"v": arrow_array}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask", string_max_length=4) + assert ct["v"].dtype == np.dtype("S4") + exported = ct.to_arrow().column("v") + assert exported.to_pylist() == [b"abcd", None, b"", b"wxyz"] + + +def test_ndarray_column_round_trip(): + arrow_array = pa.array([[1, 2], None, [3, 4]], type=pa.list_(pa.int64(), 2)) + ct, exported = round_trip(arrow_array) + assert ct["v"].null_storage == "mask" + assert exported.to_pylist() == [[1, 2], None, [3, 4]] + + +# --------------------------------------------------------------------------- +# The measurement that justifies the design +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("label", "arrow_array", "lossy_result"), + [ + # -128 is the sentinel int8 picks, so real -128 data reads back as null. + ("int8_min", pa.array([-128, None, 127], type=pa.int8()), [None, None, 127]), + # "__BLOSC2_NULL__" is literally the utf8 sentinel. + ( + "utf8_sentinel_literal", + pa.array(["", "__BLOSC2_NULL__", None], type=pa.string()), + ["", None, None], + ), + ], +) +def test_sentinel_storage_is_lossy_where_mask_storage_is_not(label, arrow_array, lossy_result): + """Silent corruption, not an error — which is what makes it worth fixing.""" + table = pa.table({"v": arrow_array}) + + sentinel = blosc2.CTable.from_arrow(table, null_storage="sentinel") + assert sentinel.to_arrow().column("v").to_pylist() == lossy_result + + masked = blosc2.CTable.from_arrow(table, null_storage="mask") + assert masked.to_arrow().column("v").to_pylist() == arrow_array.to_pylist() + + +def test_nullable_bool_imports_without_the_255_reservation(): + table = pa.table({"v": pa.array([True, None, False], type=pa.bool_())}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + assert ct["v"].dtype == np.dtype(np.bool_) + assert ct["v"][:].tolist() == [True, False, False] + assert ct["v"].is_null().tolist() == [False, True, False] + + +# --------------------------------------------------------------------------- +# Choosing the storage +# --------------------------------------------------------------------------- + + +def test_default_is_still_sentinel(): + """Phase 6 ships opt-in; the default flip is Phase 9.""" + table = pa.table({"v": pa.array([1, None, 3], type=pa.int64())}) + assert blosc2.CTable.from_arrow(table)["v"].null_storage == "sentinel" + + +def test_explicit_parameter_selects_mask(): + table = pa.table({"v": pa.array([1, None, 3], type=pa.int64())}) + assert blosc2.CTable.from_arrow(table, null_storage="mask")["v"].null_storage == "mask" + + +def test_null_policy_selects_mask(): + table = pa.table({"v": pa.array([1, None, 3], type=pa.int64())}) + with blosc2.null_policy(blosc2.NullPolicy(null_storage="mask")): + assert blosc2.CTable.from_arrow(table)["v"].null_storage == "mask" + + +def test_explicit_parameter_overrides_the_policy(): + table = pa.table({"v": pa.array([1, None, 3], type=pa.int64())}) + with blosc2.null_policy(blosc2.NullPolicy(null_storage="mask")): + ct = blosc2.CTable.from_arrow(table, null_storage="sentinel") + assert ct["v"].null_storage == "sentinel" + + +def test_column_null_values_still_forces_sentinel_per_column(): + table = pa.table( + { + "a": pa.array([1, None, 3], type=pa.int64()), + "b": pa.array([1, None, 3], type=pa.int64()), + } + ) + with blosc2.null_policy(blosc2.NullPolicy(column_null_values={"a": -7})): + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + assert ct["a"].null_storage == "sentinel" + assert ct["a"].null_value == -7 + assert ct["b"].null_storage == "mask" + + +def test_non_nullable_columns_get_no_null_channel(): + """Arrow fields are nullable by default, so this needs an explicit schema.""" + schema = pa.schema([pa.field("v", pa.int64(), nullable=False)]) + table = pa.table({"v": pa.array([1, 2, 3], type=pa.int64())}, schema=schema) + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + assert ct["v"].null_storage == "none" + + +def test_a_nullable_column_with_no_nulls_writes_no_sidecar(): + """Decision 9 survives the import path.""" + field = pa.field("v", pa.int64(), nullable=True) + table = pa.table({"v": pa.array([1, 2, 3], type=pa.int64())}, schema=pa.schema([field])) + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + assert ct["v"].null_storage == "mask" + assert ct._null_mask("v") is None + assert ct["v"].null_count() == 0 + + +def test_import_error_now_points_at_mask_storage(): + """A type with no available sentinel still fails under sentinel storage... + + ...but the message now names the way out, which is the whole point: mask + storage needs no sentinel, so every type can carry nulls. + """ + table = pa.table({"v": pa.array([1.0, None], type=pa.float64())}) + with blosc2.null_policy(blosc2.NullPolicy(float_value=None)): + with pytest.raises(TypeError, match="null_storage='mask'"): + blosc2.CTable.from_arrow(table) + + +def test_mask_storage_imports_what_no_sentinel_could(): + """The same input the previous test rejects, accepted.""" + table = pa.table({"v": pa.array([1.0, None], type=pa.float64())}) + with blosc2.null_policy(blosc2.NullPolicy(float_value=None)): + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + assert ct["v"].is_null().tolist() == [False, True] + + +def test_auto_null_sentinels_false_is_irrelevant_under_mask(): + """Mask storage picks no sentinel, so disabling the picker changes nothing.""" + table = pa.table({"v": pa.array([1, None, 3], type=pa.int64())}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask", auto_null_sentinels=False) + assert ct["v"].is_null().tolist() == [False, True, False] + + +# --------------------------------------------------------------------------- +# The utf8 validity bitmap: bit order, pinned to literal bytes +# --------------------------------------------------------------------------- + + +def test_utf8_validity_bitmap_is_lsb_first(): + """Arrow validity bitmaps are LSB-first, and a round-trip cannot prove it. + + Import would unpack a wrongly-packed bitmap the same wrong way, so this + asserts the literal buffer bytes for a known pattern instead. Rows + ``[valid, null, valid, valid, valid, null, valid, valid]`` must pack to + ``0b1101_1101`` = 0xDD, not to the MSB-first ``0b1011_1011`` = 0xBB. + """ + values = ["a", None, "c", "d", "e", None, "g", "h"] + table = pa.table({"v": pa.array(values, type=pa.string())}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + exported = ct.to_arrow().column("v").combine_chunks() + + validity = exported.buffers()[0] + assert validity is not None, "a column with nulls must carry a validity buffer" + assert validity.to_pybytes()[0] == 0xDD + + +def test_utf8_dense_buffer_export_matches_the_generic_path(): + """The dense fast path builds Arrow buffers directly; it must not diverge.""" + values = ["a", None, "c", None, "e"] + table = pa.table({"v": pa.array(values, type=pa.string())}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + + dense = ct.to_arrow().column("v") + # Deleting and re-compacting leaves a table the fast path declines (its + # guard is _last_pos == _n_rows over a dense root table), so this exercises + # the per-row path over the same data. + generic = ct.where("True" if False else ct["v"].notnull() | ct["v"].is_null()).to_arrow().column("v") + assert dense.to_pylist() == generic.to_pylist() == values + + +def test_utf8_export_of_a_filtered_view_keeps_its_nulls(): + values = ["a", None, "c", None, "e"] + table = pa.table({"v": pa.array(values, type=pa.string()), "k": pa.array([0, 1, 1, 1, 0])}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask") + view = ct.where("k == 1") + assert view.to_arrow().column("v").to_pylist() == [None, "c", None] + + +# --------------------------------------------------------------------------- +# Export of every V1 kind, from a natively-built table +# --------------------------------------------------------------------------- + + +def build(spec, values, capacity=32): + import dataclasses + + ann = ( + object + if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)) + else spec.python_type + ) + Row = dataclasses.make_dataclass("R", [("v", ann, blosc2.field(spec))]) + t = blosc2.CTable(Row, expected_size=capacity) + t.extend([(v,) for v in values]) + return t + + +EXPORT_CASES = [ + ("bool", blosc2.bool(null_storage="mask"), [True, None, False], [True, None, False]), + ("int8", blosc2.int8(null_storage="mask"), [-128, None, 127], [-128, None, 127]), + ("uint8", blosc2.uint8(null_storage="mask"), [0, None, 255], [0, None, 255]), + ( + "string", + blosc2.string(max_length=4, null_storage="mask"), + ["abcd", None, ""], + ["abcd", None, ""], + ), + ( + "bytes", + blosc2.bytes(max_length=4, null_storage="mask"), + [b"abcd", None, b""], + [b"abcd", None, b""], + ), + ( + "utf8", + blosc2.utf8(null_storage="mask"), + ["", "\x00", None], + ["", "\x00", None], + ), +] + + +@pytest.mark.parametrize(("label", "spec", "values", "expected"), EXPORT_CASES) +def test_export_carries_the_sidecar_into_arrow(label, spec, values, expected): + exported = build(spec, values).to_arrow().column("v") + assert exported.to_pylist() == expected + assert exported.null_count == 1 + + +def test_export_of_a_null_free_mask_column_has_no_nulls(): + exported = build(blosc2.int64(null_storage="mask"), [1, 2, 3]).to_arrow().column("v") + assert exported.null_count == 0 + assert exported.to_pylist() == [1, 2, 3] + + +def test_export_of_a_table_with_deletions_remaps_nulls(): + t = build(blosc2.int64(null_storage="mask"), [1, None, 3, None, 5]) + t.delete(0) + assert t.to_arrow().column("v").to_pylist() == [None, 3, None, 5] + + +def test_ndarray_export_uses_row_level_validity(): + """A sentinel ndarray column needs *every* element to match; a mask does not.""" + spec = blosc2.ndarray((2,), dtype=blosc2.int64(), null_storage="mask") + t = build(spec, [np.array([0, 0]), None, np.array([1, 2])]) + exported = t.to_arrow().column("v") + # Row 0 is all-zero — the sentinel rule's definition of null — but is a + # real value here, and only row 1 is null. + assert exported.to_pylist() == [[0, 0], None, [1, 2]] + + +# --------------------------------------------------------------------------- +# Persistence of imported nulls +# --------------------------------------------------------------------------- + + +def test_imported_nulls_survive_a_persistent_import(tmp_path): + path = str(tmp_path / "imported.b2d") + table = pa.table({"v": pa.array([1, None, 3], type=pa.int64())}) + ct = blosc2.CTable.from_arrow(table, null_storage="mask", urlpath=path, mode="w") + ct.close() + + reopened = blosc2.CTable.open(path) + try: + assert reopened["v"].null_storage == "mask" + assert reopened["v"].is_null().tolist() == [False, True, False] + assert reopened.to_arrow().column("v").to_pylist() == [1, None, 3] + finally: + reopened.close() + + +def test_import_spanning_several_batches_keeps_its_nulls(): + """The sidecar is created mid-import, on the first batch that has a null.""" + n = 300 + values = [None if i % 50 == 49 else i for i in range(n)] + table = pa.table({"v": pa.array(values, type=pa.int64())}) + batches = table.to_batches(max_chunksize=64) + ct = blosc2.CTable.from_arrow(table.schema, iter(batches), null_storage="mask") + assert ct["v"].null_count() == n // 50 + assert ct.to_arrow().column("v").to_pylist() == values From d783b00f8299ea02a9e2ca7d9fed187fb04a91f5 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 13:56:50 +0200 Subject: [PATCH 08/47] Sort, group and query mask columns by their nulls (mask-based-nulls 7) A mask column's null rows hold the column's fill, and a fill is an ordinary value to anything that only reads the values. So sort_by ordered nulls by their fill (first, ascending, inverting the nulls-last contract), group_by merged them into the genuine 0 / "" group and reduced over the fill, and where() matched them whenever the fill satisfied the leaf. Sort: the lexsort null-indicator key is now built from the sidecar, where for a sentinel it was only a refinement of the value key; the FULL-index partition reads one byte per row instead of the whole column (3.0x for U16, 1.2x for int64); utf8 rank indexes stamp nulls with null_rank, and one built before that is marked stale via null_aware, since neither O(1) signal can see the difference. sorted_slice declines for a mask column with nulls -- only the sidecar separates a null from a real fill, and the window is indexed by sorted position -- and falls back to the mask-aware sorted view. Groupby: _null_mask takes validity for value columns; a key column gets a reserved null code instead, via _CodedKeyChunk, so its nulls form a group of their own keyed None rather than joining whatever its fill was. The Cython paths defer to the generic one, which is what the dense single-key path staying (20.4 ms vs 91.8 ms on a 2M-row sum) buys back. Also the mask half of Phase 1, which had never been done: _rewrite_null_predicates and nullable_indexed both tested for NULL_SENTINEL and skipped everything else, so where("a < 500") returned every null on the scan and an ordered index over a float column returned every null for "f > 0.5", the NaN fill sorting into the range it hands back. Verified against a NumPy SQL oracle over 32 combinations. Three pre-existing storage-independent bugs fixed on the way, each reachable only because mask storage widens what a nullable column can be: descending sort of a bool column raised, descending sort of a full-range signed int put its minimum last, and groupby min/max over a bool column always answered False. Co-Authored-By: Claude Opus 5 --- plans/mask-based-nulls.md | 152 +++- src/blosc2/_utf8_array.py | 36 +- src/blosc2/ctable.py | 152 +++- src/blosc2/ctable_indexing.py | 94 ++- src/blosc2/groupby.py | 206 +++++- tests/ctable/test_null_mask_sort_groupby.py | 745 ++++++++++++++++++++ 6 files changed, 1282 insertions(+), 103 deletions(-) create mode 100644 tests/ctable/test_null_mask_sort_groupby.py diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index aad188ae4..74e985989 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,14 +1,16 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS — Phases 0–6 landed 2026-08-08.** Lossless Arrow/Parquet round-trip now -> works for every V1 kind, **opt-in** via `null_storage="mask"`; next up is Phase 7 (sort + -> groupby), then Phase 8 (migration + docs). Six premises were disproven during implementation and -> are corrected in place, each in a blockquote beside the text it corrects: the index path cannot -> be fixed by a null-aware expression (§Expression layer), the bool dtype-flip cannot move out of -> `__init__` (§Schema layer), ndarray columns do not get lazy null propagation for free -> (§Expression layer), the "free" summary min/max fast path is unsound (§Reductions), `np.packbits` -> is not needed and avoiding it is safer (§Arrow/Parquet), and `.equals()` cannot express the -> round-trip contract (§Arrow/Parquet). Drafted 2026-08-08. +> **Status: IN PROGRESS — Phases 0–7 landed 2026-08-08.** Lossless Arrow/Parquet round-trip works +> for every V1 kind and sort/groupby/query now honour a sidecar, all **opt-in** via +> `null_storage="mask"`; next up is Phase 8 (migration + docs). Six premises were disproven during +> implementation and are corrected in place, each in a blockquote beside the text it corrects: the +> index path cannot be fixed by a null-aware expression (§Expression layer), the bool dtype-flip +> cannot move out of `__init__` (§Schema layer), ndarray columns do not get lazy null propagation +> for free (§Expression layer), the "free" summary min/max fast path is unsound (§Reductions), +> `np.packbits` is not needed and avoiding it is safer (§Arrow/Parquet), and `.equals()` cannot +> express the round-trip contract (§Arrow/Parquet). Phase 7 added a seventh: **Phase 1 left the +> mask half of the query path undone** and nobody noticed until sort work went looking +> (§Expression layer, "Addendum 2"). Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -540,6 +542,44 @@ Index descriptors gain `{"null_aware": true, "null_order": "last"}` (anticipated `plans/ctable-nulls.md:614-623`, present in neither `plans/ctable-indexes-opsi.md` nor the code). Read with `.get("null_aware", False)`; bump the build token so stale indexes rebuild. +> **As built (2026-08-08).** All four items, plus one more site and two pre-existing sort bugs that +> only mask storage can reach. +> +> - **`_build_lex_keys`: the indicator key is not a refinement here, it is the whole of nulls-last.** +> For a sentinel the value key already groups the nulls together and the indicator only decides +> *where* that group goes; for a mask column the fill sorts wherever an ordinary `0` or `""` would, +> so without the key nulls came out **first** ascending and last descending — the contract exactly +> inverted, and silently. `valid[live_pos]` is one byte per row and needs no string comparison for +> `U`/`S`. +> - **`_sorted_positions_from_full_index`: real, and smaller than advertised.** The three +> partition branches (dict rank / mask / sentinel) collapsed into one shared body. Measured on 2M +> rows: **3.0x** faster for a `U16` string column (17.1 ms → 5.7 ms) but only **1.2x** for `int64` +> (8.5 → 6.9 ms). The 8x–64x is real as *bytes read*; both arrays compress well, so wall time +> tracks it only where the itemsize gap is wide. Still the best line-for-line change in the phase, +> just not by the margin the paragraph above claims. +> - **`_sorted_slice_positions` had to be found, and it bails.** Not in this section's list. The +> window read locates the null block by bisecting the *sorted values* sidecar for the null's stored +> value; under mask storage that value is the fill, which genuine rows share, and the validity +> sidecar is indexed by *physical* position where the window is indexed by sorted position — so the +> window cannot tell them apart at all. It declines for a mask column that has a sidecar and falls +> back to the full sorted view, which is now mask-aware. A null-free mask column keeps the window +> path: there is no null block to locate. +> - **`_utf8_rank_arrays` is the landmine this section promised, and the staleness rule matters more +> than the fix.** The fix is three lines (`ranks[~valid] = null_rank`, after the gather — the fill +> is a legitimate vocabulary entry other rows may share, so it has to be per row). The subtlety is +> that an index built *before* those three lines is wrong and **neither O(1) staleness signal can +> see it**: the column's row count and byte size are exactly what they were. `null_aware` absent +> from the meta is what marks it, and only for a column that has a sidecar — a sentinel column's +> nulls have always carried `null_rank`, so invalidating those would rebuild every stored utf8 +> index in the wild for nothing. +> - **Two pre-existing sort bugs, neither about nulls, both newly reachable.** The descending value +> key is built by negating the values, which breaks on the two dtypes a *nullable* column could not +> previously be: `bool` has no unary minus (`TypeError`, and this fires for a plain non-nullable +> bool column today), and a narrow signed dtype wraps on its own minimum — `-(-128)` is `-128` in +> int8 — so that row sorts as if it were the largest. A sentinel had to reserve int8's `-128` and a +> nullable bool was physically `uint8`, which is why the combination never came up. Fixed by +> negating in int64 for `b`/`u`/`i` alike. + **The indexed-OR bail (`ctable_indexing.py:1459-1461`) is not fixed by masks either** — the problem is that global post-filtering (`_exclude_null_positions:1498-1509`) drops rows that legitimately match via the *other* branch. The right fix is per-leaf and **independent of storage**: add @@ -598,6 +638,49 @@ Phase 1**, landing right after the `NullChannel` refactor and before any mask wo > comparison results to carry their null predicate through `~` — a boolean analogue of > `NullableExpr` with `__invert__` — and folds naturally into decision 8's deferred Kleene > follow-up rather than Phase 1. +> +> **Addendum 2 (2026-08-08): Phase 1 only did the sentinel half, and Phase 6 shipped the gap.** +> `_rewrite_null_predicates` tested `kind_of_spec(spec) != NULL_SENTINEL` and skipped everything +> else, so a mask column got no guard at all — and the same one-kind test in +> `ctable_indexing.py:1453` left it out of `nullable_indexed`, so no post-filter either. Both were +> measured on a 2000-row table with 1 % nulls, and both leaked: +> +> - **Scan.** `t.where("a < 500")` returned **every null**, because the int fill is `0`. Same shape +> of bug Phase 1 fixed for sentinels, read the other way round: there the stored value was a +> sentinel that happened to satisfy the leaf, here it is a fill that does. +> - **Index.** `t.where("f > 0.5")` over a float column returned **every null** even after the scan +> was fixed, because an ordered index answers by taking a range of the sorted column and the NaN +> fill sorts to the end of it. This is precisely the correction above — a null-aware expression +> cannot fix an index that never evaluates the expression — arriving a second time for a second +> storage. +> +> Fixed by giving mask storage the same two mechanisms: +> +> - the guard is `valid_pred()` injected as a `__nv{i}` operand, since there is no in-band literal +> to compare against, and **the fill stands in for the sentinel in `_sentinel_can_match`** — which +> is exactly right, being what a null row actually holds. `0` cannot satisfy `a > 10` and NaN +> cannot satisfy any ordered comparison, so those leaves stay unguarded and stay on their index; +> - a mask column with a sidecar joins `nullable_indexed`, so `_exclude_null_positions` filters it — +> reading **one byte per candidate off the sidecar**, without touching the values at all, which is +> the cheaper half of what masks buy. It stays out of `nullable_needs_exclude` for the same reason +> a NaN sentinel does: that fall-back exists for the mask-direct path, which evaluates the +> (guarded) predicate through miniexpr. +> +> Two more sites the one-kind test had hidden, both utf8, both fixed in place rather than through +> the expression layer: `_utf8_scalar_mask`/`_utf8_compare_column` compared away the sentinel string +> and so let the `""` fill through, and `utf8_span_eval` derived its per-span nulls by looking for +> the sentinel in the values — it now takes a `valids=` dict alongside `sentinels=`. A **string** +> result keeps the fill rather than a sentinel: there is nothing to write back, and a bare +> `UTF8Array` has nowhere to carry a validity channel. +> +> One drive-by fix falls out of the `partial_exact_positions` refinement block, which narrowed +> `pos` column by column while trimming the prefetched primary values only for the primary column's +> own filter: with two nullable indexed columns and nulls in the non-primary one, `prefetched` came +> out **misaligned with `candidates`**. Rewritten as one combined keep-mask applied once, which is +> both correct and shorter. +> +> Verified against a NumPy SQL oracle over 32 combinations — {mask, sentinel} × {indexed, +> unindexed} × 8 expression shapes including `|` and `~` — all agreeing exactly. ### Groupby @@ -611,6 +694,40 @@ after `__setitem__`** — budget accordingly. One semantic improvement follows from decision 6: NaN in a float *value* column is no longer missing. Keep the `is_key` NaN coercion at `:2131-2134` for keys (dropna semantics). +> **As built (2026-08-08).** It was not the messiest integration — that was the *key* side, which +> this section does not mention at all. +> +> - **A mask key column needs recoding, not a threaded flag.** Threading `valid=` fixes value +> columns, and that part is as described (`_null_mask` grows the kwarg; the generic path and the +> dense single-key path gather validity by the same `live_mask` as the values). But a *key* column +> has a second problem no validity flag solves: with `dropna=False` the null rows have to form a +> group of their own, and their fill is a value a genuine row may hold, so they would merge into +> the `0` group of an int key or the `""` group of a string one. The fix is to give them a reserved +> code — exactly what a dictionary column gets for free from `null_code`. `_Utf8KeyChunk` was +> already the right shape for that, so it became `_CodedKeyChunk` with a `null_code` field, and +> `_coded_chunk_with_nulls` recodes any mask key chunk into one. `uniques[null_code] is None`, so +> the null group comes out keyed **`None`** and a mask-storage output column writes it back as a +> real null — where a sentinel column can only offer its sentinel (`group_by(dropna=False)` over +> one returns a group keyed `-1`). +> - **`_null_output_value` returns `None` for a mask output spec**, which is the same point one layer +> up: a group with no non-null input no longer has to come back as `0` and hope nobody reads it as +> data. Note `sum`'s output spec is a fresh non-nullable `float64`/`int64`, so *that* aggregate +> still spells missing as `NaN` — unchanged, and the same for both storages. +> - **The fast paths bail, but not all of them, and the difference is 4.5x.** Every path reads +> nullity out of the values; a Cython kernel is handed a `skip_nan` flag, not a validity array, so +> the four Cython paths defer to the generic path whenever a mask column in play holds a null. The +> dense single-int-key path is plain NumPy and already routes value columns through `_null_mask`, +> so it only needed the sidecar — and keeping it matters: 2M-row `sum` grouped by an int key went +> 91.8 ms → 20.4 ms once it stayed, against 10.8 ms for the sentinel equivalent. A mask *key* +> column still has to leave it, since a `_CodedKeyChunk` is not the array of dense non-negative +> ints that path indexes with. Threading validity into the kernels is the named follow-up. +> - **One pre-existing bug, storage-independent.** `min`/`max` seed a per-group accumulator with the +> dtype's opposite identity, and `_max_identity`/`_min_identity` had no `bool` case — so +> `np.full(n, None, dtype=bool)` gave `False`, a min accumulator could never rise above it, and +> every all-`True` group reduced to `False`. Reachable today with a plain non-nullable bool column +> on any generic-path aggregation (a string key is enough); a nullable one was `uint8`, whose +> identities are fine, which is why mask storage is what surfaced it. + ### Nullable-bool cleanup Under masks, `bool(nullable=True)` yields physical `np.bool_` — no `uint8`, no reserved `255`. @@ -728,7 +845,7 @@ default-created tables require them. | 4 | ✅ **Read/write + null API.** `extend`/`append`/`_coerce_row_to_storage`/`__setitem__`/`assign`; `is_null`/`notnull`/`null_count`/`fillna`/`_nonnull_chunks`/`to_numpy(masked=)`/`dropna`; **plus every gather-and-rebuild path** (`sort_by` ×3, `take`, `slice`), which this section had not listed. Mask columns fully usable. `tests/ctable/test_null_mask_api.py` (90 tests). | **L** | **High** (turned out to be the reference cycle, not `__setitem__`) | | 5 | ✅ **Expressions + reductions.** `_ndarray_values_for_reduction`, argmin/argmax (both were reducing over the *fill*), `_reduction_null_mask` as the one storage-agnostic entry point. `_raw_null_pred`/`_lazy_nonnull_mask`/`_is_nullable_bool` needed nothing — Phases 0–4 had already made them storage-agnostic. **The ndarray-propagation gain is not real and was not done**, and the free summary fast path is unsound; both corrections are above. `tests/ctable/test_null_mask_expressions.py` (39 tests). | S (was M) | Low (was Med) | | 6 | ✅ **Arrow/Parquet.** Import + export for all V1 kinds; `arrow_slice(valid=)`; `null_storage=` on `from_arrow`/`from_parquet`; the "no sentinel available" import error now names the way out instead of being deleted (it still fires for sentinel storage, which still cannot represent those types). No `packbits` — pyarrow's own packing is borrowed instead. Ships **opt-in**; the default stays `"sentinel"`. `tests/ctable/test_null_mask_arrow.py` (42 tests). | M | Med | -| 7 | **Sort + groupby.** `_build_lex_keys`, `_sorted_positions_from_full_index` (big I/O win), `_utf8_rank_arrays(valid=)`, groupby `_null_mask` threading. | M | Med-High | +| 7 | ✅ **Sort + groupby.** `_build_lex_keys` (the indicator key is nulls-last *entirely*, not a refinement), `_sorted_positions_from_full_index` (3.0x for `U16`, 1.2x for `int64` — the I/O win is in bytes, not proportionally in time), `_utf8_rank_arrays(valid=)` plus a `null_aware` staleness rule no O(1) signal could replace, `_sorted_slice_positions` bails, groupby `_null_mask(valid=)` **plus `_CodedKeyChunk`**, which this section had not anticipated: a mask *key* column needs a reserved null code, not a threaded flag. **Plus the mask half of Phase 1**, which had never been done — `where()` leaked nulls on both the scan and the index (see §Expression layer, Addendum 2). Three pre-existing storage-independent bugs fixed on the way: descending sort of `bool` (raised) and of full-range signed ints (wrong order), and groupby `min`/`max` over `bool` (always `False`). `tests/ctable/test_null_mask_sort_groupby.py` (75 tests). | **L** (was M) | Med-High | | 8 | **Migration + docs.** `convert_nulls`, `Column.null_storage`, `info()`, `doc/reference/ctable.rst` null-policy rewrite, release notes. | S–M | Low | | 9 | **Default flips to `"mask"`.** A one-line `NullPolicy` change plus release notes — lossless round-trip is why the default exists. Lands **no earlier than one release after Phase 6** so older readers in the wild already understand schema version 3. | S | Low | | 10 | **Index null-awareness remainder** *(independent)*. Mask-aware summary builder; `null_aware`/`null_order` descriptors; re-enable `_summary_minmax_source` for mask and sentinel columns alike. | **L** | High | @@ -736,3 +853,18 @@ default-created tables require them. The riskiest, most-coupled work is isolated into Phases 4, 7 and 10, each of which can slip without blocking the others. Phase 9 is a policy change, not code — its only prerequisite is that Phases 2–8 have soaked for a release. + +## Named follow-ups (not blocking any phase) + +- **Validity through the Cython groupby kernels.** A mask column holding a null costs ~1.9x on a + 2M-row grouped `sum` against the sentinel equivalent, because the four Cython paths bail; the + kernels take a `skip_nan` flag where they would need a `values_valid` array. Two of them + (`groupby_hash_i64x2_f64`, `groupby_dense_int_count_checked`) already accept one, so this is + partly a matter of using what is there. +- **A mask *key* column back on the dense single-key path.** It leaves because + `_CodedKeyChunk.codes` are chunk-local, not the dense global ints that path indexes with. +- **`__setitem__`'s fast paths** for mask columns (deferred in Phase 4, still unmeasured). +- **Kleene three-valued logic**, decision 8 — which now also owns the operator-form negation leak + pinned as a `strict=True` xfail in `tests/ctable/test_null_predicate_rewrite.py`. +- **Phase 10** as listed above: mask-aware summary builder, `null_aware`/`null_order` descriptors, + and with them a genuinely indexed `OR` over a nullable column. diff --git a/src/blosc2/_utf8_array.py b/src/blosc2/_utf8_array.py index 4dabac4ce..a0d4b4017 100644 --- a/src/blosc2/_utf8_array.py +++ b/src/blosc2/_utf8_array.py @@ -126,6 +126,19 @@ def utf8_spans(arrays: dict, n_logical: int, span_rows: int, budget: int): yield a, min(a + rows, stop) +def _span_null_mask(raw: np.ndarray, sentinel, valid, start: int, stop: int) -> np.ndarray | None: + """Which rows of one span are null, or ``None`` when the operand has no nulls. + + Whichever channel the operand uses: a sentinel found among *raw*'s values, + or a slice of a mask-storage column's validity sidecar. + """ + if sentinel is not None: + return raw == sentinel + if valid is not None: + return ~np.asarray(valid[start:stop], dtype=bool) + return None + + def utf8_span_eval( expr: str, operands: dict, @@ -133,6 +146,7 @@ def utf8_span_eval( sentinels: dict, n_phys: int, *, + valids: dict | None = None, strict: bool = False, span_rows: int = UTF8_EXPR_SPAN, budget: int = UTF8_EXPR_BUDGET, @@ -155,6 +169,14 @@ def utf8_span_eval( result therefore needs the operands to agree on one sentinel, and raises ``ValueError`` when they do not. + A **mask-storage** operand has no sentinel to find in its values -- its null + rows already hold the ``""`` fill, which other rows may legitimately share -- + so it names its validity sidecar in *valids* instead (physical, ``True`` = + not null). Either channel feeds the same per-span *nulls*, so a boolean + result is forced ``False`` for those rows just the same. A **string** result + keeps the fill: there is no sentinel to write back, and a bare + :class:`UTF8Array` has nowhere to carry a validity channel of its own. + Span operands are handed over as blosc2 arrays rather than NumPy ones: the NumPy route evaluates through ``slices_eval``, which never reaches miniexpr, so the string kernels would be bypassed for correct-looking @@ -179,18 +201,18 @@ def utf8_span_eval( nulls = None for name, arr in arrays.items(): raw = np.asarray(arr[start:stop]) - nv = sentinels[name] - if nv is not None: - mask = raw == nv - if mask.any(): - nulls = mask if nulls is None else (nulls | mask) - raw = np.where(mask, "", raw) + mask = _span_null_mask( + raw, sentinels[name], None if valids is None else valids.get(name), start, stop + ) + if mask is not None and mask.any(): + nulls = mask if nulls is None else (nulls | mask) + raw = np.where(mask, "", raw) span[name] = blosc2.asarray(raw.astype(utf8_span_dtype(raw))) res = np.asarray(blosc2.lazyexpr(expr, span).compute(**compute_kwargs)) if nulls is not None: if res.dtype.kind == "b": res = res & ~nulls - elif res.dtype.kind == "U": + elif res.dtype.kind == "U" and null_value is not None: # The sentinel may be wider than the computed values. res = np.where( nulls, diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 6f6055e0f..758ff5815 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -2231,6 +2231,11 @@ def _utf8_compare_column(self, numpy_op, other: Column): other_arr = other._raw_col nv = self.null_value other_nv = other.null_value + # Mask storage keeps nullity out of the values, so the side channel is + # the only thing that can exclude a null here; the fill it left behind + # ("" for utf8) is an ordinary string that compares like any other. + valid = self._nulls.valid_array() + other_valid = other._nulls.valid_array() def fn(chunk, start, stop): rhs = other_arr[start:stop] @@ -2239,6 +2244,10 @@ def fn(chunk, start, stop): res &= chunk != nv if other_nv is not None: res &= rhs != other_nv + if valid is not None: + res &= np.asarray(valid[start:stop], dtype=bool) + if other_valid is not None: + res &= np.asarray(other_valid[start:stop], dtype=bool) return res raw = self._utf8_chunked_bool(fn) @@ -2254,6 +2263,11 @@ def _utf8_scalar_mask(self, numpy_op, value: str) -> np.ndarray: see :meth:`_utf8_compare_scalar` for that. """ nv = self.null_value + # Under mask storage there is no sentinel string to compare away: a null + # row holds the empty-string fill, which is a perfectly ordinary value + # here (and one other rows may legitimately share). The sidecar is the + # only thing that can tell the two apart. + valid = self._nulls.valid_array() indexed = self._utf8_index_mask(numpy_op, value) if indexed is not None: @@ -2267,6 +2281,8 @@ def fn(arr, start, stop): res = ~res if nv is not None: res &= ~arr.equal_mask_span(nv, start, stop) + if valid is not None: + res &= np.asarray(valid[start:stop], dtype=bool) return res else: @@ -2282,6 +2298,8 @@ def fn(arr, start, stop): res = ~lt if nv is not None: res = res & ~arr.equal_mask_span(nv, start, stop) + if valid is not None: + res = res & np.asarray(valid[start:stop], dtype=bool) return res return self._utf8_chunked_bytes(fn) @@ -4555,6 +4573,13 @@ def _utf8_rank_index_stale(self, name: str, utf8_rank_meta: dict) -> bool: col = self._root_table._cols.get(name) if col is None: return True + if not utf8_rank_meta.get("null_aware", False) and self._root_table._null_mask(name) is not None: + # Built before mask nulls were stamped with null_rank, so its ranks + # place them wherever the empty-string fill factorized -- rank 0, + # where a genuine "" is indistinguishable from a null. The column's + # bytes need not have changed for that to be wrong, so neither + # signal below can catch it: rebuild rather than trust it. + return True return len(col) != utf8_rank_meta.get("n_rows") or int(col._bytes_used) != utf8_rank_meta.get( "nbytes" ) @@ -12117,6 +12142,7 @@ def _sorted_positions_from_full_index(self, name: str, ascending: bool) -> np.nd null_value = None null_code = None + valid_arr = None is_dict_rank = False if name in root._cols: col_info = root._schema.columns_by_name.get(name) @@ -12124,6 +12150,8 @@ def _sorted_positions_from_full_index(self, name: str, ascending: bool) -> np.nd null_value = getattr(col_info.spec, "null_value", None) if isinstance(col_info.spec, DictionarySpec): null_code = col_info.spec.null_code + elif getattr(col_info.spec, "uses_mask", False): + valid_arr = root._null_mask(name) descriptor = catalog.get(name) if descriptor is None or descriptor.get("kind") != "full" or descriptor.get("stale", False): descriptor = None @@ -12196,40 +12224,39 @@ def _sorted_positions_from_full_index(self, name: str, ascending: bool) -> np.nd current_valid = self._valid_rows[:] positions = positions[current_valid[positions]] + # The index sorts by stored value, but sort_by's contract is nulls-last. + # Partition explicitly so it holds for either order and for any way of + # spelling a null -- a NaN sentinel sorts last on its own, an integer + # sentinel like INT64_MIN sorts first, and a mask column's fill sorts + # wherever an ordinary 0 or "" would. Free each 24M-element temporary + # as soon as it is consumed to keep peak memory near the size of the + # permutation itself. + null_phys = None if is_dict_rank: - # Dict-rank index: positions sorted by rank (int32), nulls have sentinel null_rank. - # Partition null rows using codes (int32), not decoded strings. + # Positions are sorted by rank (int32), nulls carrying null_rank. + # Partition using the codes, not the decoded strings. codes = np.asarray(root._cols[name].codes[:], dtype=np.int32) null_phys = codes == null_code del codes - if null_phys.any(): - is_null = null_phys[positions] - del null_phys - nulls = positions[is_null] - nonnull = positions[~is_null] - del is_null, positions - if not ascending: - nonnull = nonnull[::-1] - return np.concatenate([nonnull, nulls]) - # No nulls: fall through to simple reverse + elif valid_arr is not None: + # Mask storage: one byte per row off the sidecar rather than the + # whole raw column, which is where this path used to spend most of + # its I/O -- 8x for int64, 64x for a U16 string column. + null_phys = ~np.asarray(valid_arr[:]) elif null_value is not None: - # The index sorts by raw value, but sort_by's contract is nulls-last. - # Partition explicitly so it holds for any sentinel (NaN sorts last, - # an integer sentinel like INT64_MIN sorts first) and either order. - # Free each 24M-element temporary as soon as it is consumed to keep - # peak memory near the size of the permutation itself. raw = np.asarray(root._cols[name][:]) null_phys = sentinel_mask(raw, null_value) del raw - if null_phys.any(): - is_null = null_phys[positions] - del null_phys - nulls = positions[is_null] - nonnull = positions[~is_null] - del is_null, positions - if not ascending: - nonnull = nonnull[::-1] - return np.concatenate([nonnull, nulls]) + + if null_phys is not None and null_phys.any(): + is_null = null_phys[positions] + del null_phys + nulls = positions[is_null] + nonnull = positions[~is_null] + del is_null, positions + if not ascending: + nonnull = nonnull[::-1] + return np.concatenate([nonnull, nulls]) if not ascending: positions = positions[::-1] @@ -12287,6 +12314,11 @@ def _build_lex_keys( else: raw = gathered[name] if name in gathered else self._cols[name][live_pos] nv = getattr(col_info.spec, "null_value", None) if col_info else None + valid = ( + self._null_mask(name) + if col_info is not None and getattr(col_info.spec, "uses_mask", False) + else None + ) # Value key if not asc: @@ -12294,7 +12326,14 @@ def _build_lex_keys( # strings can't be negated — invert via rank rank = np.argsort(np.argsort(raw, kind="stable"), kind="stable") lex_keys.append((n - 1 - rank).astype(np.intp)) - elif np.issubdtype(raw.dtype, np.unsignedinteger): + elif raw.dtype.kind in "bui": + # Negate in int64, never in the column's own dtype. bool has + # no unary minus at all, and a narrow signed dtype wraps on + # its own minimum -- ``-(-128)`` is ``-128`` in int8 -- which + # silently leaves that row sorted as if it were the largest. + # Mask storage is what made both reachable in a *nullable* + # column: a sentinel had to reserve int8's -128, and a + # nullable bool was physically uint8. lex_keys.append(-raw.astype(np.int64)) else: lex_keys.append(-raw) @@ -12305,6 +12344,13 @@ def _build_lex_keys( # so nulls always sort last (0 before 1 → non-null before null). if is_dict_col and col_info.spec.nullable: lex_keys.append(is_null.astype(np.intp)) + elif valid is not None: + # Mask storage: the fill occupying a null slot is an ordinary + # value as far as lexsort is concerned (0 and "" both sort + # first), so this key is the whole of nulls-last here -- not + # just a refinement of the value key as it is for a sentinel. + # One byte per row, and no string comparison for U/S. + lex_keys.append((~np.asarray(valid[live_pos])).astype(np.intp)) elif nv is not None: lex_keys.append(sentinel_mask(raw, nv).astype(np.intp)) @@ -12457,6 +12503,20 @@ def _sorted_slice_positions(self, name: str, ascending: bool, key: slice) -> np. return None col_info = self._schema.columns_by_name.get(name) + if ( + col_info is not None + and getattr(col_info.spec, "uses_mask", False) + and self._null_mask(name) is not None + ): + # Mask storage with at least one null: the null rows hold this + # column's fill, which is an ordinary value in the sorted sidecar, + # so bisecting for it would sweep up genuine fill-valued rows as + # well. Only the sidecar can tell the two apart, and it is indexed + # by physical position, not by sorted position, so the window read + # cannot consult it. Fall back to the full sorted view, which is + # mask-aware. A mask column that has never held a null keeps this + # path: with no sidecar there is no null block to locate. + return None null_value = getattr(col_info.spec, "null_value", None) if col_info is not None else None # Rank index: the sidecar holds int32 ranks, so the null block is located # by null_rank, not by the column's own sentinel (a string, for utf8). @@ -13662,10 +13722,17 @@ def _rewrite_null_predicates( Each nullable column referenced by the expression contributes a validity operand (``a != null_value``, or ``~isnan(a)`` for a NaN - sentinel), which :func:`~blosc2.ctable_nulls.rewrite_null_predicates` - conjoins onto every comparison that reads it. The validity operand is - a lazy expression over the same raw array, so it fuses into the same - pass rather than materializing anything. + sentinel; the ``.notnull`` sidecar for a mask-storage column), which + :func:`~blosc2.ctable_nulls.rewrite_null_predicates` conjoins onto every + comparison that reads it. The validity operand is a lazy expression + over the same raw array, so it fuses into the same pass rather than + materializing anything. + + A **mask-storage** column needs this just as much as a sentinel one, and + for the same reason read the other way round: its nulls hold the + column's fill, and a fill is a value like any other to miniexpr, so + ``a < 10`` over an int column matched every null (fill ``0``) before + this ran. This makes the *scan* path correct — including the scan that an indexed OR over a nullable column bails to. It does not replace the @@ -13684,10 +13751,28 @@ def _rewrite_null_predicates( if name in self._computed_cols: continue col_info = self._schema.columns_by_name.get(name) - if col_info is None or kind_of_spec(col_info.spec) != NULL_SENTINEL: + if col_info is None: + continue + kind = kind_of_spec(col_info.spec) + if kind not in (NULL_SENTINEL, NULL_MASK): continue if not self._expression_references_name(expr, name): continue + if kind == NULL_MASK: + # No in-band value to compare against, so the guard can only be + # the sidecar, injected as an operand. The *fill* stands in for + # the sentinel in the can-match test, and correctly so: it is + # what a null row actually holds, so a leaf the fill cannot + # satisfy cannot return a null either. That keeps ``a > 10`` on + # an int column (fill 0) and every ordered comparison on a float + # one (fill NaN) unguarded, and so still on their index. + valid_pred = self[name]._nulls.valid_pred() + if valid_pred is None: + continue # no sidecar: the column has never held a null + guard = f"__nv{i}" + new_operands[guard] = valid_pred + valid_exprs[name] = (guard, fill_value_for(col_info.spec)) + continue guard = sentinel_guard_expr(name, col_info.spec.null_value) if guard is None: # A sentinel with no literal form: fall back to an injected @@ -13935,6 +14020,9 @@ def _utf8_span_eval( {name: self._cols[col_of(name, name)] for name in utf8_names}, {name: self[col_of(name, name)].null_value for name in utf8_names}, len(self._valid_rows), + # A mask-storage operand carries no sentinel for the driver to find, + # so it hands over its validity sidecar instead. + valids={name: self._null_mask(col_of(name, name)) for name in utf8_names}, strict=strict, span_rows=self._UTF8_EXPR_SPAN, budget=self._UTF8_EXPR_BUDGET, diff --git a/src/blosc2/ctable_indexing.py b/src/blosc2/ctable_indexing.py index 38a7f47ae..84291ed67 100644 --- a/src/blosc2/ctable_indexing.py +++ b/src/blosc2/ctable_indexing.py @@ -20,7 +20,7 @@ import blosc2 from blosc2 import compute_chunks_blocks -from blosc2.ctable_nulls import NULL_SENTINEL, is_nan_sentinel, kind_of_spec, sentinel_mask +from blosc2.ctable_nulls import NULL_MASK, NULL_SENTINEL, is_nan_sentinel, kind_of_spec, sentinel_mask from blosc2.schema import ( DictionarySpec, ListSpec, @@ -97,7 +97,7 @@ def _dict_rank_hash(dictionary) -> str: _UTF8_RANK_SPAN = 1 << 20 -def _utf8_rank_arrays(col, n_phys: int, null_value: str | None): +def _utf8_rank_arrays(col, n_phys: int, null_value: str | None, *, valid=None): """Alphabetical rank per row for a utf8 column, plus its staleness metadata. Sorting by rank is sorting by decoded string, so an ``int32`` rank column @@ -106,9 +106,17 @@ def _utf8_rank_arrays(col, n_phys: int, null_value: str | None): there is no stored code array, so the column is factorized here; the factorizer hashes raw bytes and only ever decodes the distinct values. - Null rows carry a sentinel *string*, so the sentinel is just another - vocabulary entry; it is given the largest rank so nulls sort last, matching - both the dictionary index and ``_build_lex_keys``. + Under sentinel storage null rows carry a sentinel *string*, so the sentinel + is just another vocabulary entry; it is given the largest rank so nulls sort + last, matching both the dictionary index and ``_build_lex_keys``. + + Under mask storage there is **no sentinel in the vocabulary**: a null row + holds the empty-string fill, which factorizes as an ordinary entry with rank + 0 — so nulls would sort *first* and be indistinguishable from genuine ``""`` + rows in every rank comparison the index answers. *valid* (physical, one + flag per row) is what closes that hole: null rows are stamped with + ``null_rank`` after the rank gather, exactly where the sentinel path would + have put them. """ fact = col.factorizer() codes = np.empty(n_phys, dtype=np.int64) @@ -131,12 +139,18 @@ def _utf8_rank_arrays(col, n_phys: int, null_value: str | None): # turns a query literal into a rank (np.searchsorted) without touching data. sorted_vocab = uniques[order] ranks = code_to_rank[codes] if n_entries else np.zeros(n_phys, dtype=np.int32) + if valid is not None: + # Mask storage: after the gather, not before. The fill in a null row is + # a legitimate vocabulary entry that other rows may share, so the + # rewrite has to be per row rather than per vocabulary entry. + ranks = np.where(np.asarray(valid[:n_phys], dtype=bool), ranks, null_rank) # Staleness signals must be O(1) to check: re-deriving the vocabulary would # mean factorizing the column again on every query. Any write already marks # every index stale, so these only have to catch a rebuilt-but-changed # column, for which row count plus blob size is enough. meta = { "null_rank": int(null_rank), + "null_aware": null_value is not None or valid is not None, "vocab_len": int(n_entries), "n_rows": int(n_phys), "nbytes": int(col._bytes_used), @@ -897,7 +911,7 @@ def create_index( # noqa: C901 # same length _utf8_rank_index_stale() compares the meta against. n_phys = len(col_arr) ranks_arr, utf8_rank_meta, utf8_vocab = _utf8_rank_arrays( - col_arr, n_phys, self[col_name].null_value + col_arr, n_phys, self[col_name].null_value, valid=self._null_mask(col_name) ) col_arr = blosc2.asarray(ranks_arr) @@ -1450,18 +1464,29 @@ def _try_index_where(self, expr_result: blosc2.LazyExpr) -> np.ndarray | None: # a NaN sentinel sorts last, so every NaN row comes back for any # ``>`` query. The expression being correct therefore does not make the # index result correct, and these positions must still be filtered. - nullable_indexed = [ - name - for name, _arr, _descriptor in indexed_columns - if kind_of_spec(root._schema.columns_by_name[name].spec) == NULL_SENTINEL - ] + # Mask storage is in the same position as a sentinel here, with the fill + # playing the sentinel's part: the ordered range that answers ``f > 0.5`` + # contains the NaN fill of every null row, so a mask column's nulls come + # back too unless they are filtered out. A mask column with no sidecar + # has no nulls at all, so there is nothing for it to contribute. + null_kinds = {} + for name, _arr, _descriptor in indexed_columns: + kind = kind_of_spec(root._schema.columns_by_name[name].spec) + if kind == NULL_SENTINEL or (kind == NULL_MASK and root._null_mask(name) is not None): + null_kinds[name] = kind + nullable_indexed = list(null_kinds) # Only non-NaN sentinels need the *positions* fall-back below; for a NaN # sentinel the mask-direct path can stay, because that path evaluates the - # predicate through miniexpr rather than reading an ordered range. + # predicate through miniexpr rather than reading an ordered range. A + # mask column is in the same position: whatever its fill, the predicate + # miniexpr evaluates carries the sidecar guard that + # ``CTable._rewrite_null_predicates`` conjoined onto every leaf the fill + # could have satisfied. nullable_needs_exclude = [ name - for name in nullable_indexed - if not is_nan_sentinel(root._schema.columns_by_name[name].spec.null_value) + for name, kind in null_kinds.items() + if kind == NULL_SENTINEL + and not is_nan_sentinel(root._schema.columns_by_name[name].spec.null_value) ] # Global null post-filtering is not correct for OR expressions: it would @@ -1510,10 +1535,17 @@ def _try_index_where(self, expr_result: blosc2.LazyExpr) -> np.ndarray | None: def _exclude_null_positions(positions): positions = np.asarray(positions, dtype=np.int64) - for name in nullable_indexed: - nv = root._schema.columns_by_name[name].spec.null_value - raw = root._cols[name][positions] - positions = positions[~sentinel_mask(raw, nv)] + for name, kind in null_kinds.items(): + if positions.size == 0: + break + if kind == NULL_MASK: + # One byte per candidate off the sidecar, and the values are + # not read at all -- the cheaper half of what masks buy. + keep = np.asarray(root._null_mask(name)[positions], dtype=bool) + else: + nv = root._schema.columns_by_name[name].spec.null_value + keep = ~sentinel_mask(root._cols[name][positions], nv) + positions = positions[keep] return positions if plan.exact_positions is not None: @@ -1548,17 +1580,23 @@ def _exclude_null_positions(positions): if nullable_indexed and primary_op_name is not None: raw = primary_col_arr[candidates] raw = np.asarray(raw) if hasattr(raw, "__array__") else raw - pos = candidates - for name in nullable_indexed: + # One combined keep-mask over the full candidate set, so the + # prefetched primary values stay aligned with the positions + # however many columns contribute nulls. Narrowing the + # positions column by column while trimming ``raw`` only for the + # primary would silently misalign the two. + keep = np.ones(len(candidates), dtype=bool) + for name, kind in null_kinds.items(): + if kind == NULL_MASK: + keep &= np.asarray(root._null_mask(name)[candidates], dtype=bool) + continue nv = root._schema.columns_by_name[name].spec.null_value - if name == primary_col_name: - keep = ~sentinel_mask(raw, nv) - pos = pos[keep] - raw = raw[keep] # already filtered for refinement reuse - else: - pos = pos[~sentinel_mask(root._cols[name][pos], nv)] - candidates = pos - prefetched = {primary_op_name: raw} + values = raw if name == primary_col_name else root._cols[name][candidates] + keep &= ~sentinel_mask(values, nv) + candidates = candidates[keep] + # Reuse the primary read for refinement, saving a second sparse + # gather -- but only when it survived as an aligned array. + prefetched = {primary_op_name: raw[keep]} if isinstance(raw, np.ndarray) else None else: candidates = _exclude_null_positions(candidates) diff --git a/src/blosc2/groupby.py b/src/blosc2/groupby.py index d3ec84d6c..f9fa8e31e 100644 --- a/src/blosc2/groupby.py +++ b/src/blosc2/groupby.py @@ -55,24 +55,31 @@ class _AggState: @dataclasses.dataclass -class _Utf8KeyChunk: - """A utf8 key-column chunk, factorized to chunk-local integer codes. - - ``codes[i]`` indexes ``uniques`` (a ``StringDType`` array sorted - ascending), so null detection, live-row masking, and per-chunk - ``np.unique`` all run on int64 codes; only the (few) distinct strings are - ever decoded. Produced by :meth:`CTableGroupBy._read_key_chunk` via - ``UTF8Array.factorizer``. +class _CodedKeyChunk: + """A key-column chunk factorized to chunk-local integer codes. + + ``codes[i]`` indexes ``uniques``, so null detection, live-row masking, and + per-chunk ``np.unique`` all run on int64 codes; for a utf8 column, only the + (few) distinct strings are ever decoded. Produced by + :meth:`CTableGroupBy._read_key_chunk` -- for utf8 via + ``UTF8Array.factorizer``, whose ``uniques`` are ``StringDType`` sorted + ascending, and for a mask-storage column via :func:`_coded_chunk_with_nulls`. + + *null_code* is the code reserved for null rows, or ``-1`` when this chunk's + nulls are in band (a sentinel string among the ``uniques``). When it is set, + ``uniques[null_code] is None``, so the null group is displayed and written + as a null rather than as whatever value the storage left in those slots. """ codes: np.ndarray uniques: np.ndarray + null_code: int = -1 def __len__(self) -> int: return len(self.codes) - def take(self, mask: np.ndarray) -> _Utf8KeyChunk: - return _Utf8KeyChunk(self.codes[mask], self.uniques) + def take(self, mask: np.ndarray) -> _CodedKeyChunk: + return _CodedKeyChunk(self.codes[mask], self.uniques, self.null_code) def code_of(self, value: str) -> int: """Code of *value* in this chunk, or -1 when absent (uniques are sorted).""" @@ -82,6 +89,37 @@ def code_of(self, value: str) -> int: return -1 +def _coded_chunk_with_nulls(chunk, valid: np.ndarray) -> _CodedKeyChunk: + """Recode a mask-storage key chunk so its nulls get a code of their own. + + A mask-storage column has no value that *means* null: its null rows hold the + column's fill, and genuine rows may hold the same thing, so grouping on the + values alone silently merges the two -- every null row landing in the ``0`` + group of an int key, or the ``""`` group of a string one. Recoding to + chunk-local integers with one reserved code is what a dictionary column gets + for free from its ``null_code``; this gives it to the other kinds. + + The reserved code maps to ``None`` in the vocabulary, so with + ``dropna=False`` the null group comes out keyed ``None`` -- which a + mask-storage output column can write back as a real null, where a sentinel + column can only offer its sentinel. + """ + if isinstance(chunk, _CodedKeyChunk): + # Already coded by the utf8 factorizer; the null rows only need to be + # moved off their fill's code and onto one of their own. + codes, uniques = chunk.codes, chunk.uniques + else: + uniques, inverse = np.unique(chunk[valid], return_inverse=True) + codes = np.empty(len(chunk), dtype=np.int64) + codes[valid] = inverse + null_code = len(uniques) + vocab = np.empty(null_code + 1, dtype=object) + vocab[:null_code] = list(uniques) + vocab[null_code] = None + codes = np.where(valid, codes, null_code) + return _CodedKeyChunk(codes, vocab, null_code) + + def _is_column_like(value: Any) -> bool: return isinstance(getattr(value, "_col_name", None), str) @@ -481,10 +519,25 @@ def _try_fast_paths(self, specs: list[_AggSpec], use_arg_positions: bool): UDF aggregations always fall through to the generic chunked path below, which is the only one that accumulates raw per-group values instead of a mergeable scalar state. + + The **Cython** paths do too whenever a mask-storage column in play holds + a null: each reads nullity out of the values -- a kernel is told a + ``skip_nan`` flag, not given a validity array -- and a mask column's + values say nothing about which rows are null. Threading a sidecar + through six kernels is a performance follow-up; getting the answer right + is not. The dense single-key path is plain NumPy and already routes its + value columns through :meth:`_null_mask`, so a mask *value* column need + only hand over its sidecar there; only a mask *key* column has to leave, + since its recoded chunk is a :class:`_CodedKeyChunk` rather than the + array of dense non-negative ints that path indexes with. + + A mask column with no sidecar has never held a null and stays on every + fast path. """ if any(s.op == "udf" for s in specs): return None - if not use_arg_positions: + mask_null = self._mask_null_columns(specs) + if not use_arg_positions and not mask_null: for attempt in ( self._try_execute_cython_dense_int_key, self._try_execute_cython_two_int_key_hash, @@ -499,10 +552,11 @@ def _try_fast_paths(self, specs: list[_AggSpec], use_arg_positions: bool): # before the generic float hash path, which is markedly slower. Unlike # the Cython kernels above, it tracks row positions, so it also serves # argmin/argmax — keeping them off the slow generic hash+merge path. - fast = self._try_execute_dense_single_int_key(specs) - if fast is not None: - return fast - if not use_arg_positions: + if not any(name in self.keys for name in mask_null): + fast = self._try_execute_dense_single_int_key(specs) + if fast is not None: + return fast + if not use_arg_positions and not mask_null: return self._try_execute_cython_float_hash(specs) return None @@ -538,7 +592,7 @@ def _execute_with_result_target(self, specs: list[_AggSpec]): keys_live = [ values.take(live_mask) - if isinstance(values, _Utf8KeyChunk) + if isinstance(values, _CodedKeyChunk) else np.asarray(values)[live_mask] for values in raw_keys ] @@ -550,9 +604,16 @@ def _execute_with_result_target(self, specs: list[_AggSpec]): value_chunks = { name: np.asarray(self.table._cols[name][start:stop])[live_mask] for name in value_cols } + # Gathered by the same live_mask as the values, so a validity flag + # stays beside the row it belongs to. + valid_chunks = { + name: valid_slice[live_mask] + for name in value_cols + if (valid_slice := self._valid_chunk(name, start, stop)) is not None + } partials = self._compute_partials( - specs, unique_keys, inverse, value_chunks, logical_positions[live_mask] + specs, unique_keys, inverse, value_chunks, logical_positions[live_mask], valid_chunks ) display_keys = self._display_keys(unique_keys) normalized_keys = self._normalized_keys(display_keys) @@ -1420,6 +1481,13 @@ def ensure_size(size: int) -> bool: value_chunks = { name: np.asarray(self.table._cols[name][start:stop])[live_mask] for name in value_cols } + # Gathered by the same live_mask as the values, so a validity flag + # stays beside the row it belongs to. + valid_chunks = { + name: valid_slice[live_mask] + for name in value_cols + if (valid_slice := self._valid_chunk(name, start, stop)) is not None + } row_positions = logical_chunk[live_mask] if need_positions else None for spec in specs: @@ -1428,7 +1496,12 @@ def ensure_size(size: int) -> bool: continue assert spec.input_col is not None values = value_chunks[spec.input_col] - non_null = ~self._null_mask(spec.input_col, values, is_key=False) + non_null = ~self._null_mask( + spec.input_col, + values, + is_key=False, + valid=valid_chunks.get(spec.input_col), + ) if spec.op == "count": states[spec.output_col] += np.bincount( keys, weights=non_null.astype(np.int64), minlength=len(present) @@ -1559,7 +1632,48 @@ def _chunk_size(self) -> int: # column). return base * -(-target // base) + def _mask_null_columns(self, specs: list[_AggSpec]) -> list[str]: + """The key and value columns in play that have a validity sidecar. + + Non-empty means this aggregation has to take the generic chunked path; + see :meth:`_try_fast_paths`. Only mask-storage columns are probed, and + only those the schema names -- an absent sidecar is the common case and + means the column has never held a null. + """ + names = [*self.keys, *(s.input_col for s in specs if s.input_col is not None)] + candidates = set(self.table._null_mask_names) + return [ + name + for name in dict.fromkeys(names) + if name in candidates and self.table._null_mask(name) is not None + ] + + def _valid_chunk(self, name: str, start: int, stop: int) -> np.ndarray | None: + """Physical validity for rows ``[start, stop)`` of *name*, or ``None``. + + ``None`` is the answer for every column whose nulls travel in band with + its values, and equally for a mask-storage column that has never held a + null -- both mean "nothing here to look up on the side". Groupby reads + physical windows throughout, so the sidecar slices straight across. + """ + mask = self.table._null_mask(name) + return None if mask is None else np.asarray(mask[start:stop], dtype=bool) + def _read_key_chunk(self, name: str, start: int, stop: int) -> np.ndarray: + chunk = self._read_raw_key_chunk(name, start, stop) + valid = self._valid_chunk(name, start, stop) + if valid is None: + return chunk + if getattr(chunk, "dtype", None) is not None and chunk.dtype.kind == "f": + # A float *key* keeps NaN-as-missing so dropna stays predictable, + # as it does for a sentinel column -- see _null_mask. Folding it in + # here puts NaN keys in the same group as the nulls rather than + # leaving _null_mask to re-derive it from recoded values it can no + # longer see. + valid = valid & ~np.isnan(chunk) + return _coded_chunk_with_nulls(chunk, valid) + + def _read_raw_key_chunk(self, name: str, start: int, stop: int) -> np.ndarray: col_info = self.table._schema.columns_by_name[name] if self.table._is_dictionary_column(col_info): return np.asarray(self.table._cols[name].codes[start:stop], dtype=np.int32) @@ -1587,7 +1701,7 @@ def _read_key_chunk(self, name: str, start: int, stop: int) -> np.ndarray: codes = rank[codes] if len(order) else codes if stop > n: codes = np.concatenate([codes, np.zeros(stop - max(start, n), dtype=np.int64)]) - return _Utf8KeyChunk(codes, uniques[order]) + return _CodedKeyChunk(codes, uniques[order]) return np.asarray(self.table._cols[name][start:stop]) def _factorize_keys( @@ -1595,7 +1709,7 @@ def _factorize_keys( ) -> tuple[np.ndarray | list[np.ndarray], np.ndarray]: if len(keys_live) == 1: arr = keys_live[0] - if isinstance(arr, _Utf8KeyChunk): + if isinstance(arr, _CodedKeyChunk): # The chunk is already factorized to dense string-rank codes; # dedupe them with an O(n) bincount instead of a sort. The # np.unique contract — uniques ascending by string — holds @@ -1613,7 +1727,7 @@ def _factorize_keys( # order); the codes in the deduped rows are mapped back to strings # below, into object fields (StringDType cannot be a structured-array # field). - pack_arrs = [arr.codes if isinstance(arr, _Utf8KeyChunk) else arr for arr in keys_live] + pack_arrs = [arr.codes if isinstance(arr, _CodedKeyChunk) else arr for arr in keys_live] composite = self._composite_int_factorize(pack_arrs) if composite is not None: unique_fields, inverse = composite @@ -1625,13 +1739,13 @@ def _factorize_keys( packed_unique, inverse = np.unique(packed, return_inverse=True) unique_fields = [packed_unique[f"k{i}"] for i in range(len(pack_arrs))] out_dtype = [ - (f"k{i}", object if isinstance(arr, _Utf8KeyChunk) else pack_arrs[i].dtype) + (f"k{i}", object if isinstance(arr, _CodedKeyChunk) else pack_arrs[i].dtype) for i, arr in enumerate(keys_live) ] unique = np.empty(len(unique_fields[0]), dtype=out_dtype) for i, arr in enumerate(keys_live): field = unique_fields[i] - unique[f"k{i}"] = arr.uniques[field] if isinstance(arr, _Utf8KeyChunk) else field + unique[f"k{i}"] = arr.uniques[field] if isinstance(arr, _CodedKeyChunk) else field return unique, inverse @staticmethod @@ -1745,6 +1859,7 @@ def _compute_partials( inverse: np.ndarray, value_chunks: dict[str, np.ndarray], row_positions: np.ndarray, + valid_chunks: dict[str, np.ndarray] | None = None, ) -> dict[str, Any]: n_groups = len(unique_keys) partials: dict[str, Any] = {} @@ -1755,7 +1870,12 @@ def _compute_partials( assert spec.input_col is not None values = value_chunks[spec.input_col] - non_null = ~self._null_mask(spec.input_col, values, is_key=False) + non_null = ~self._null_mask( + spec.input_col, + values, + is_key=False, + valid=None if valid_chunks is None else valid_chunks.get(spec.input_col), + ) if spec.op == "count": partials[spec.output_col] = np.bincount( @@ -2114,11 +2234,28 @@ def _result_spec_for_agg(self, spec: _AggSpec) -> SchemaSpec: return float64() return copy.deepcopy(input_spec) - def _null_mask(self, name: str, values: np.ndarray, *, is_key: bool) -> np.ndarray: + def _null_mask( + self, name: str, values: np.ndarray, *, is_key: bool, valid: np.ndarray | None = None + ) -> np.ndarray: + """True where this chunk of *name* is null, one flag per row. + + *values* is the chunk the caller already read, which is enough for the + in-band kinds. A mask-storage column's nullity is not in its values at + all, so it arrives separately: as *valid* for a value column, or already + folded into a reserved code for a key column (:meth:`_read_key_chunk`). + """ col_info = self.table._schema.columns_by_name[name] spec = col_info.spec null_value = getattr(spec, "null_value", None) - if isinstance(values, _Utf8KeyChunk): + if valid is not None: + # The fill under a null must not be tested as a value: a 0 fill + # would look non-null and a NaN one doubly null. The sidecar is the + # whole answer here, and for a *value* column that includes leaving + # a genuine NaN alone -- in a mask column NaN is a value. + return ~np.asarray(valid, dtype=bool) + if isinstance(values, _CodedKeyChunk): + if values.null_code >= 0: + return values.codes == values.null_code if null_value is None: return np.zeros(len(values), dtype=bool) return values.codes == values.code_of(null_value) @@ -2214,6 +2351,12 @@ def _python_type_for_spec(spec: SchemaSpec): def _max_identity(dtype: np.dtype): dtype = np.dtype(dtype) + if dtype.kind == "b": + # bool needs its own case: np.full(n, None, dtype=bool) is *False*, so a + # min accumulator seeded with it can never rise above False and every + # all-True group reduced to False. (Storage-independent -- reachable + # with any plain bool column on the generic chunked path.) + return True if dtype.kind in "iu": return np.iinfo(dtype).max if dtype.kind == "f": @@ -2225,6 +2368,8 @@ def _max_identity(dtype: np.dtype): def _min_identity(dtype: np.dtype): dtype = np.dtype(dtype) + if dtype.kind == "b": + return False # see _max_identity if dtype.kind in "iu": return np.iinfo(dtype).min if dtype.kind == "f": @@ -2235,7 +2380,16 @@ def _min_identity(dtype: np.dtype): def _null_output_value(spec: SchemaSpec): + """The value to write for a group whose aggregate is missing. + + A mask-storage output column answers ``None``: nullity goes in its sidecar, + so the result is a *real* null rather than a value standing in for one -- + which is the whole reason a group with no non-null input no longer has to + come back as ``0`` or ``NaN`` and hope nobody reads it as data. + """ dtype = getattr(spec, "dtype", None) + if getattr(spec, "uses_mask", False): + return None null_value = getattr(spec, "null_value", None) if null_value is not None: return null_value diff --git a/tests/ctable/test_null_mask_sort_groupby.py b/tests/ctable/test_null_mask_sort_groupby.py new file mode 100644 index 000000000..f9fcc5b51 --- /dev/null +++ b/tests/ctable/test_null_mask_sort_groupby.py @@ -0,0 +1,745 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Sort, group_by and query paths over mask-storage columns (Phase 7). + +Every path here had the same shape of bug, and it is worth stating once: a +mask column's null rows hold the column's *fill*, and a fill is an ordinary +value to anything that only looks at the values. So + +* ``sort_by`` sorted nulls by their fill — first for an ascending int column + (``0``), first for a string one (``""``) — where the nulls-last contract and + every sentinel column put them last; +* ``group_by`` merged the nulls into the genuine ``0`` / ``""`` group, and + reduced over the fill instead of skipping it; +* ``where("a < 10")`` matched every null, because ``0 < 10``; +* a ``FULL`` index on a float column returned every null for ``f > 0.5``, + because the NaN fill sorts into the ordered range the index hands back. + +The tests are written as a differential oracle against sentinel storage: the +same logical data both ways, asserted to give the same answer. That is the +strongest form available, because the sentinel path has been correct since +Phase 1 and is independently tested. Where the two are *supposed* to differ, +the divergence is asserted directly instead. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 + + +def annotation_for(spec): + if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)): + return object + return spec.python_type + + +def table(rows, capacity=64, urlpath=None, **cols): + Row = dataclasses.make_dataclass( + "SortRow", [(n, annotation_for(s), blosc2.field(s)) for n, s in cols.items()] + ) + kwargs = {"urlpath": str(urlpath), "mode": "w"} if urlpath is not None else {} + t = blosc2.CTable(Row, expected_size=max(capacity, len(rows)), **kwargs) + if rows: + t.extend(rows) + return t + + +def one_col(values, spec, capacity=64, urlpath=None): + return table([(v,) for v in values], capacity=capacity, urlpath=urlpath, a=spec) + + +#: The V1 kinds a sort key can be, each with a mask spec, a sentinel spec, and +#: the value the sentinel path has to write where the mask path writes ``None``. +KEY_KINDS = [ + ("int64", blosc2.int64, {}, -(2**62)), + ("float64", blosc2.float64, {}, np.nan), + ("string", blosc2.string, {"max_length": 4}, "\x7f\x7f"), + ("utf8", blosc2.utf8, {}, "__BLOSC2_NULL__"), +] + + +def pair(values, factory, kw, sentinel): + """The same logical *values* as a mask table and a sentinel table.""" + mask = one_col(values, factory(null_storage="mask", **kw)) + raw = [sentinel if v is None else v for v in values] + sent = one_col(raw, factory(nullable=True, null_value=sentinel, **kw)) + return mask, sent + + +def as_list(col): + """A column's live values with nulls spelled ``None``.""" + null = col.is_null() + return [None if null[i] else _scalar(col[i]) for i in range(len(col))] + + +def _scalar(value): + return value.item() if hasattr(value, "item") else value + + +# --------------------------------------------------------------------------- +# sort_by: nulls last, in both directions, for every kind +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("label", "factory", "kw", "sentinel"), KEY_KINDS) +@pytest.mark.parametrize("ascending", [True, False]) +def test_sort_by_puts_mask_nulls_last(label, factory, kw, sentinel, ascending): + """The contract sentinel columns already keep: nulls last, either direction. + + Before this phase the null-indicator lexsort key was only built for a + sentinel, so a mask column sorted its nulls by the fill — ``0`` and ``""`` + both sort *first* ascending, which silently reversed the contract. + """ + values = ["e", None, "a", "z", None, "b"] if label in ("string", "utf8") else [5, None, 1, 9, None, 2] + mask, sent = pair(values, factory, kw, sentinel) + + got = as_list(mask.sort_by("a", ascending=ascending)["a"]) + n_nulls = sum(v is None for v in values) + assert got[-n_nulls:] == [None] * n_nulls + assert got[:-n_nulls] == sorted((v for v in values if v is not None), reverse=not ascending) + # And the same answer the sentinel path gives, null spelling aside. + assert got == as_list(sent.sort_by("a", ascending=ascending)["a"]) + + +def test_sort_by_null_free_mask_column_needs_no_indicator_key(): + """No sidecar, no nulls, no extra key — decision 9 all the way through.""" + t = one_col([3, 1, 2], blosc2.int64(null_storage="mask")) + assert t._null_mask("a") is None + assert as_list(t.sort_by("a")["a"]) == [1, 2, 3] + + +def test_sort_by_multi_key_orders_nulls_last_per_key(): + t = table( + [(1, "b"), (1, None), (None, "a"), (1, "a"), (None, None)], + a=blosc2.int64(null_storage="mask"), + s=blosc2.string(max_length=2, null_storage="mask"), + ) + st = t.sort_by(["a", "s"]) + assert as_list(st["a"]) == [1, 1, 1, None, None] + assert as_list(st["s"]) == ["a", "b", None, "a", None] + + +def test_sort_by_inplace_keeps_values_with_their_nulls(): + t = one_col([5, None, 1], blosc2.int64(null_storage="mask")) + t.sort_by("a", inplace=True) + assert as_list(t["a"]) == [1, 5, None] + + +def test_sort_by_view_orders_nulls_last(): + t = one_col([5, None, 1], blosc2.int64(null_storage="mask")) + assert as_list(t.sort_by("a", view=True)["a"]) == [1, 5, None] + + +@pytest.mark.parametrize( + ("spec", "values", "expected"), + [ + (blosc2.int8(null_storage="mask"), [3, None, -128, 7], [7, 3, -128, None]), + (blosc2.uint8(null_storage="mask"), [3, None, 255, 0], [255, 3, 0, None]), + (blosc2.bool(null_storage="mask"), [True, None, False], [True, False, None]), + (blosc2.bytes(max_length=2, null_storage="mask"), [b"b", None, b""], [b"b", b"", None]), + ], +) +def test_descending_sort_over_a_full_range_mask_column(spec, values, expected): + """Two storage-independent sort bugs that only mask storage can reach. + + The descending value key is built by negating the values, and that broke on + the two dtypes a nullable column could not previously *be*: a ``bool`` + column has no unary minus (a nullable bool used to be physically ``uint8``), + and a narrow signed dtype wraps on its own minimum -- ``-(-128) == -128`` in + int8 -- so that row sorted as if it were the largest. A sentinel had to + reserve int8's ``-128``, so no nullable int8 column could hold it. + """ + assert as_list(one_col(values, spec).sort_by("a", ascending=False)["a"]) == expected + + +def test_descending_sort_of_a_plain_bool_column(): + """The same fix, stated for the case that was broken with no nulls at all.""" + t = one_col([True, False, True], blosc2.bool()) + assert as_list(t.sort_by("a", ascending=False)["a"]) == [True, True, False] + + +# --------------------------------------------------------------------------- +# The FULL-index sort path reads the sidecar, not the whole column +# --------------------------------------------------------------------------- + + +def indexed_pair(values, factory, kw, sentinel, tmp_path): + mask = one_col(values, factory(null_storage="mask", **kw), urlpath=tmp_path / "m.b2t") + raw = [sentinel if v is None else v for v in values] + sent = one_col(raw, factory(nullable=True, null_value=sentinel, **kw), urlpath=tmp_path / "s.b2t") + for t in (mask, sent): + t.create_index("a", kind="full") + return mask, sent + + +@pytest.mark.parametrize(("label", "factory", "kw", "sentinel"), KEY_KINDS) +@pytest.mark.parametrize("ascending", [True, False]) +def test_full_index_sort_matches_the_lexsort(label, factory, kw, sentinel, ascending, tmp_path): + """A FULL index sorts by stored value, so nulls have to be repartitioned. + + Which is not new — the sentinel path already did it — but a mask column had + no branch, so its nulls came back wherever the fill sorted. + """ + if label in ("string", "utf8"): + pool = ["alfa", "beta", "", "zeta", "gam"] # within string's max_length=4 + else: + pool = [5, 1, 9, 2, 7] + values = [None if i % 7 == 0 else pool[i % len(pool)] for i in range(200)] + mask, sent = indexed_pair(values, factory, kw, sentinel, tmp_path) + + got = as_list(mask.sort_by("a", ascending=ascending)["a"]) + assert got == as_list(sent.sort_by("a", ascending=ascending)["a"]) + n_nulls = sum(v is None for v in values) + assert got[-n_nulls:] == [None] * n_nulls + + +def whole_column_reads(t, name, monkeypatch, body): + """Run *body*, returning which arrays *t* read whole (``arr[:]``).""" + seen = [] + real_getitem = blosc2.NDArray.__getitem__ + values_arr = t._cols[name] + sidecar = t._null_mask(name) + + def counting(self, key): + if isinstance(key, slice) and key == slice(None): + if self is values_arr: + seen.append("values") + elif sidecar is not None and self is sidecar: + seen.append("sidecar") + return real_getitem(self, key) + + monkeypatch.setattr(blosc2.NDArray, "__getitem__", counting) + body() + monkeypatch.undo() + return seen + + +def test_full_index_sort_reads_the_sidecar_instead_of_the_values(tmp_path, monkeypatch): + """The highest value-per-line change in the phase, measured as such. + + Locating the null rows in a FULL index's permutation means reading a whole + column: the sentinel path reads the *values* to compare against the + sentinel, 8 bytes a row for ``int64`` and 64 for a ``U16`` string. A mask + column reads its sidecar instead — one byte a row, and bool NDArrays + compress to almost nothing. + """ + values = [None if i % 5 == 0 else i for i in range(200)] + mask = one_col(values, blosc2.int64(null_storage="mask"), urlpath=tmp_path / "m.b2t") + sent = one_col( + [-1 if v is None else v for v in values], + blosc2.int64(nullable=True, null_value=-1), + urlpath=tmp_path / "s.b2t", + ) + for t in (mask, sent): + t.create_index("a", kind="full") + + assert whole_column_reads(mask, "a", monkeypatch, lambda: mask.sort_by("a")) == ["sidecar"] + assert whole_column_reads(sent, "a", monkeypatch, lambda: sent.sort_by("a")) == ["values"] + + +def test_sorted_slice_falls_back_when_the_column_has_nulls(tmp_path): + """The window read cannot locate a mask column's null block. + + It finds the null block by bisecting the *sorted values* sidecar for the + null's stored value; under mask storage that value is the fill, which + genuine rows share, and the validity sidecar is indexed by physical + position rather than sorted position so the window cannot consult it. So + it declines, and the full sorted view — which is mask-aware — answers. + """ + values = [None if i % 9 == 0 else (i % 17) for i in range(300)] + t = one_col(values, blosc2.int64(null_storage="mask"), urlpath=tmp_path / "m.b2t") + t.create_index("a", kind="full") + + assert t._sorted_slice_positions("a", True, slice(0, 5)) is None + for key in (slice(0, 5), slice(-5, None), slice(280, 300)): + for ascending in (True, False): + window = as_list(t.sorted_slice("a", key, ascending=ascending)["a"]) + full = as_list(t.sort_by("a", ascending=ascending, view=True)[key]["a"]) + assert window == full + + +def test_sorted_slice_keeps_its_window_read_when_there_are_no_nulls(tmp_path): + """A mask column with no sidecar has no null block to locate.""" + t = one_col(list(range(300)), blosc2.int64(null_storage="mask"), urlpath=tmp_path / "m.b2t") + t.create_index("a", kind="full") + assert t._sorted_slice_positions("a", True, slice(0, 5)) is not None + + +# --------------------------------------------------------------------------- +# utf8 rank index: the fill must not factorize as an ordinary value +# --------------------------------------------------------------------------- + + +def test_utf8_rank_index_separates_nulls_from_genuine_empty_strings(tmp_path): + """The landmine this phase was warned about, and it is a real one. + + A utf8 rank index factorizes the column; under mask storage the ``""`` fill + is just another vocabulary entry, and it factorizes to **rank 0** — the + smallest — so nulls both sorted first and answered ``a == ''`` alongside + the rows that really are empty. + """ + values = ["b", None, "", "a", None, ""] + t = one_col(values, blosc2.utf8(null_storage="mask"), urlpath=tmp_path / "m.b2t") + t.create_index("a", kind="full") + + assert len(t.where("a == ''")) == 2 # the two real empty strings, not the nulls + assert as_list(t.where("a == ''")["a"]) == ["", ""] + assert as_list(t.sort_by("a")["a"]) == ["", "", "a", "b", None, None] + + +def test_utf8_rank_arrays_stamps_nulls_with_the_null_rank(): + """Directly, since this is where the recoding happens.""" + from blosc2.ctable_indexing import _utf8_rank_arrays + + t = one_col(["b", None, "", "a"], blosc2.utf8(null_storage="mask")) + col = t._cols["a"] + valid = t._null_mask("a") + ranks, meta, vocab = _utf8_rank_arrays(col, 4, None, valid=valid) + assert meta["null_aware"] is True + assert list(vocab) == ["", "a", "b"] + assert int(ranks[1]) == meta["null_rank"] == 3 + assert int(ranks[2]) == 0 # the genuine "" keeps rank 0 + + +def test_a_pre_mask_utf8_rank_index_is_treated_as_stale(tmp_path): + """An index built before nulls got their own rank cannot be trusted. + + Neither of the O(1) staleness signals can catch it — the column's row count + and byte size are exactly what they were — so the absence of ``null_aware`` + from the meta is what marks it, and only for a column that has a sidecar. + """ + t = one_col(["b", None, ""], blosc2.utf8(null_storage="mask"), urlpath=tmp_path / "m.b2t") + t.create_index("a", kind="full") + meta = t._get_index_catalog()["a"]["full"]["utf8_rank"] + assert t._utf8_rank_index_stale("a", meta) is False + + del meta["null_aware"] + assert t._utf8_rank_index_stale("a", meta) is True + + +def test_a_sentinel_utf8_rank_index_stays_fresh_without_the_flag(tmp_path): + """The staleness rule must not fire for sentinel columns. + + Their nulls have always carried ``null_rank``, so an index with no + ``null_aware`` flag is merely an older one, not a wrong one — invalidating + it would rebuild every stored utf8 index in the wild for nothing. + """ + t = one_col( + ["b", "__BLOSC2_NULL__", ""], + blosc2.utf8(nullable=True), + urlpath=tmp_path / "s.b2t", + ) + t.create_index("a", kind="full") + meta = dict(t._get_index_catalog()["a"]["full"]["utf8_rank"]) + meta.pop("null_aware", None) + assert t._utf8_rank_index_stale("a", meta) is False + + +# --------------------------------------------------------------------------- +# where(): the string form, indexed and not +# --------------------------------------------------------------------------- + +WHERE_CASES = [ + "a < 500", + "a > 500", + "a != 7", + "a == 3", + "~(a > 500)", +] + + +def numeric_pair(tmp_path, indexed): + rng = np.random.default_rng(5) + values = [None if i % 31 == 0 else int(rng.integers(0, 1000)) for i in range(2000)] + mask = one_col(values, blosc2.int64(null_storage="mask"), urlpath=tmp_path / f"m{indexed}.b2t") + sent = one_col( + [-1 if v is None else v for v in values], + blosc2.int64(nullable=True, null_value=-1), + urlpath=tmp_path / f"s{indexed}.b2t", + ) + if indexed: + mask.create_index("a", kind="full") + sent.create_index("a", kind="full") + return mask, sent + + +@pytest.mark.parametrize("query", WHERE_CASES) +@pytest.mark.parametrize("indexed", [False, True]) +def test_string_predicates_reject_mask_nulls(query, indexed, tmp_path): + """``a < 10`` matched every null before this: the int fill is ``0``. + + The operator form was already correct (``_null_aware_compare`` collapses + null to False at the leaf); it was only the string form that compared the + stored fill, exactly as it used to compare the stored sentinel before + Phase 1. + """ + mask, sent = numeric_pair(tmp_path, indexed) + got = mask.where(query) + assert got["a"].null_count() == 0 + assert len(got) == len(sent.where(query)) + + +@pytest.mark.parametrize("indexed", [False, True]) +def test_or_over_a_mask_column_keeps_the_other_branch(indexed, tmp_path): + """The per-leaf guard is what makes OR right; a global filter would not be. + + A row null in ``a`` but matching ``b`` must survive ``(a > x) | (b < y)``. + """ + rng = np.random.default_rng(6) + n = 2000 + a = [None if i % 31 == 0 else int(rng.integers(0, 1000)) for i in range(n)] + b = [int(rng.integers(0, 1000)) for _ in range(n)] + t = table( + list(zip(a, b, strict=True)), + capacity=n, + urlpath=tmp_path / f"or{indexed}.b2t", + a=blosc2.int64(null_storage="mask"), + b=blosc2.int64(), + ) + if indexed: + t.create_index("a", kind="full") + + expected = sum(((av is not None and av > 800) or bv < 200) for av, bv in zip(a, b, strict=True)) + assert len(t.where("(a > 800) | (b < 200)")) == expected + + +def test_a_nan_fill_does_not_come_back_from_an_ordered_index(tmp_path): + """The index failure this phase found, and the one masks make worse. + + An ordered index answers ``f > 0.5`` by taking a *range of the sorted + column* — it never evaluates the predicate — and NaN sorts to the end of + that range. A mask float column's fill is NaN, so every null row was + returned. Making the expression null-aware cannot fix this; the + positions have to be filtered, which is what a mask column now joins + ``nullable_indexed`` to get. + """ + rng = np.random.default_rng(9) + values = [None if i % 23 == 0 else float(rng.random()) for i in range(2000)] + t = one_col(values, blosc2.float64(null_storage="mask"), urlpath=tmp_path / "f.b2t") + unindexed = len(t.where("a > 0.5")) + t.create_index("a", kind="full") + indexed = t.where("a > 0.5") + assert indexed["a"].null_count() == 0 + assert len(indexed) == unindexed + + +def test_null_free_mask_column_stays_out_of_the_null_bookkeeping(tmp_path): + """No sidecar means no guard operand and no post-filter to pay for.""" + t = one_col(list(range(2000)), blosc2.int64(null_storage="mask"), urlpath=tmp_path / "n.b2t") + t.create_index("a", kind="full") + rewritten, operands = t._rewrite_null_predicates("a < 500", {"a": t._cols["a"]}) + assert rewritten == "a < 500" + assert list(operands) == ["a"] + assert len(t.where("a < 500")) == 500 + + +def test_a_guard_is_only_emitted_where_the_fill_could_match(): + """Keeping needless guards out is what keeps a query on its index. + + The fill takes the sentinel's place in the can-match test, and it is the + right stand-in: it is what a null row actually holds. ``0`` cannot satisfy + ``a > 10``, so that leaf is left alone. + """ + t = one_col([1, None, 20], blosc2.int64(null_storage="mask")) + operands = {"a": t._cols["a"]} + assert t._rewrite_null_predicates("a > 10", operands)[0] == "a > 10" + guarded, extended = t._rewrite_null_predicates("a < 10", operands) + assert guarded != "a < 10" + assert any(name.startswith("__nv") for name in extended) + + +# --------------------------------------------------------------------------- +# utf8 comparisons and the span driver +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "query", + ["a < 'b'", "a > 'b'", "a != 'a'", "a == ''", "startswith(a, 'a')"], +) +def test_utf8_predicates_agree_between_storages(query): + values = ["e", None, "a", "z", "", "b"] + mask, sent = pair(values, blosc2.utf8, {}, "__BLOSC2_NULL__") + assert as_list(mask.where(query)["a"]) == as_list(sent.where(query)["a"]) + + +def test_utf8_span_driver_excludes_mask_nulls(): + """The span driver materializes nulls to ``""`` and re-applies nullity. + + It found them by comparing against the sentinel, which a mask column does + not have — so a boolean result over the ``""`` fill came back True for + nulls whenever the fill satisfied the expression. + """ + t = one_col(["ax", None, "bx"], blosc2.utf8(null_storage="mask")) + assert as_list(t.where("startswith(a, '')")["a"]) == ["ax", "bx"] + + +def test_utf8_column_vs_column_comparison_excludes_nulls(): + t = table( + [("a", "a"), (None, "a"), ("b", None), ("c", "c")], + a=blosc2.utf8(null_storage="mask"), + b=blosc2.utf8(null_storage="mask"), + ) + assert as_list(t.where(t["a"] == t["b"])["a"]) == ["a", "c"] + + +# --------------------------------------------------------------------------- +# group_by: as a key and as a value +# --------------------------------------------------------------------------- + +AGGS = { + "cnt": ("v", "count"), + "sm": ("v", "sum"), + "mn": ("v", "min"), + "mx": ("v", "max"), + "av": ("v", "mean"), +} + + +def grouped(t, keys, dropna): + """``{key tuple -> {agg -> value}}``, nulls spelled ``None`` throughout.""" + g = t.group_by(keys, dropna=dropna, sort=True).agg(**AGGS) + out = {} + for i in range(len(g)): + key = tuple(as_list(g[name])[i] for name in keys) + row = {} + for name in AGGS: + value = as_list(g[name])[i] + if isinstance(value, float) and np.isnan(value): + value = None # a non-nullable float output spells "missing" NaN + row[name] = value + out[key] = row + return out + + +def groupby_pair(): + rng = np.random.default_rng(11) + n = 400 + k = [None if i % 13 == 0 else int(rng.integers(0, 4)) for i in range(n)] + s = [None if i % 17 == 0 else ["aa", "bb", "cc"][int(rng.integers(0, 3))] for i in range(n)] + v = [None if i % 7 == 0 else float(rng.integers(0, 100)) for i in range(n)] + mask = table( + list(zip(k, s, v, strict=True)), + capacity=n, + k=blosc2.int64(null_storage="mask"), + s=blosc2.string(max_length=2, null_storage="mask"), + v=blosc2.float64(null_storage="mask"), + ) + sent = table( + [ + (-1 if a is None else a, "ZZ" if b is None else b, np.nan if c is None else c) + for a, b, c in zip(k, s, v, strict=True) + ], + capacity=n, + k=blosc2.int64(nullable=True, null_value=-1), + s=blosc2.string(max_length=2, nullable=True, null_value="ZZ"), + v=blosc2.float64(nullable=True), + ) + return mask, sent + + +@pytest.mark.parametrize("keys", [["k"], ["s"], ["k", "s"]]) +@pytest.mark.parametrize("dropna", [True, False]) +def test_group_by_agrees_with_sentinel_storage(keys, dropna, tmp_path): + """The differential oracle for group_by, keys and values together. + + A mask key column used to merge its nulls into the group of whatever its + fill was — ``0`` for the int key, ``""`` for the string one — and a mask + value column reduced over the fill, turning any group containing a null + into ``NaN`` for ``sum``/``mean``. + """ + mask, sent = groupby_pair() + got = grouped(mask, keys, dropna) + expected = { + tuple(None if part in (-1, "ZZ") else part for part in key): row + for key, row in grouped(sent, keys, dropna).items() + } + assert got == expected + + +def test_group_by_a_mask_key_keeps_nulls_out_of_the_fill_group(): + """Stated on its own, because it is the failure that is easiest to miss. + + ``0`` is a perfectly ordinary key, so a null row landing in its group + inflates a real answer rather than producing an obviously wrong one. + """ + t = table( + [(0, 1.0), (None, 2.0), (0, 4.0)], + k=blosc2.int64(null_storage="mask"), + v=blosc2.float64(null_storage="mask"), + ) + g = t.group_by(["k"], dropna=True).agg(total=("v", "sum")) + assert as_list(g["k"]) == [0] + assert as_list(g["total"]) == [5.0] + + +def test_group_by_a_mask_key_writes_its_null_group_as_a_null(): + """With ``dropna=False`` the null group's key is a *real* null. + + A sentinel column can only offer its sentinel here, which is why + ``group_by(dropna=False)`` over one returns a group keyed ``-1``. Mask + storage can say what it means. + """ + t = table( + [(1, 1.0), (None, 2.0), (None, 4.0)], + k=blosc2.int64(null_storage="mask"), + v=blosc2.float64(null_storage="mask"), + ) + g = t.group_by(["k"], dropna=False, sort=True).agg(total=("v", "sum")) + assert as_list(g["k"]) == [None, 1] + assert as_list(g["total"]) == [6.0, 1.0] + + +def test_group_by_a_mask_value_skips_nulls_not_fills(): + t = table( + [("x", 3), ("x", None), ("y", None)], + s=blosc2.string(max_length=1), + v=blosc2.int64(null_storage="mask"), + ) + g = t.group_by(["s"], sort=True).agg(n=("v", "count"), lo=("v", "min")) + assert as_list(g["n"]) == [1, 0] + # min over a group with no non-null value is a null, not the 0 fill. + assert as_list(g["lo"]) == [3, None] + + +def test_group_by_a_mask_float_value_treats_nan_as_a_value(): + """Decision 6, in the one place it is observable in an aggregate. + + A sentinel float column's NaN *is* its null, so ``sum`` skips it. A mask + column's NaN is data, so it propagates — which is Arrow's answer and + NumPy's. + """ + t = table( + [("x", 1.0), ("x", float("nan")), ("x", None)], + s=blosc2.string(max_length=1), + v=blosc2.float64(null_storage="mask"), + ) + g = t.group_by(["s"]).agg(total=("v", "sum"), n=("v", "count")) + assert np.isnan(g["total"][0]) + assert as_list(g["n"]) == [2] # the NaN counts; the null does not + + +def test_group_by_a_mask_float_key_groups_nan_with_the_nulls(): + """Keys keep NaN-as-missing, so ``dropna`` stays predictable. + + This is the one place a float mask column does *not* follow decision 6, and + deliberately: the rule is about values, and a key that sometimes forms its + own NaN group and sometimes does not would make ``dropna`` unusable. It is + also what the sentinel path does. + """ + t = table( + [(1.0, 1), (float("nan"), 2), (None, 4)], + k=blosc2.float64(null_storage="mask"), + v=blosc2.int64(), + ) + assert len(t.group_by(["k"], dropna=True).agg(total=("v", "sum"))) == 1 + g = t.group_by(["k"], dropna=False, sort=True).agg(total=("v", "sum")) + assert as_list(g["total"]) == [6, 1] + + +def test_group_by_utf8_mask_key_separates_nulls_from_empty_strings(): + t = table( + [("", 1), (None, 2), ("", 4)], + s=blosc2.utf8(null_storage="mask"), + v=blosc2.int64(), + ) + g = t.group_by(["s"], dropna=False, sort=True).agg(total=("v", "sum")) + assert as_list(g["s"]) == [None, ""] + assert as_list(g["total"]) == [2, 5] + + +@pytest.mark.parametrize( + ("spec", "values"), + [ + (blosc2.int8(null_storage="mask"), [3, None, -128, 3]), + (blosc2.uint8(null_storage="mask"), [255, None, 0, 255]), + (blosc2.bool(null_storage="mask"), [True, None, False, True]), + (blosc2.bytes(max_length=2, null_storage="mask"), [b"aa", None, b"", b"aa"]), + (blosc2.utf8(null_storage="mask"), ["aa", None, "", "aa"]), + ], +) +def test_group_by_a_mask_key_of_every_v1_kind(spec, values): + """Each kind's fill is a value some real row could hold, so each needs the recode.""" + t = table([(v, i) for i, v in enumerate(values)], k=spec, v=blosc2.int64()) + distinct = {v for v in values if v is not None} + dropped = t.group_by(["k"], dropna=True, sort=True).agg(total=("v", "sum")) + assert len(dropped) == len(distinct) + assert None not in as_list(dropped["k"]) + kept = t.group_by(["k"], dropna=False, sort=True).agg(total=("v", "sum")) + assert as_list(kept["k"]).count(None) == 1 + assert sum(as_list(kept["total"])) == sum(range(len(values))) + + +def test_group_by_min_over_a_bool_value_column(): + """A storage-independent bug the mask path is what routed into. + + ``min``/``max`` over a group seed an accumulator with the dtype's opposite + identity, and ``bool`` had none -- ``np.full(n, None, dtype=bool)`` is + ``False``, so a min accumulator could never rise above it and every all-True + group reduced to ``False``. Reachable with a plain non-nullable bool column + on any generic-path aggregation; a nullable one used to be ``uint8``, whose + identities are fine, which is why mask storage surfaced it. + """ + t = table( + [("a", True), ("a", True), ("b", False), ("b", True)], + k=blosc2.string(max_length=1), + v=blosc2.bool(), + ) + g = t.group_by(["k"], sort=True).agg(lo=("v", "min"), hi=("v", "max")) + assert as_list(g["lo"]) == [True, False] + assert as_list(g["hi"]) == [True, True] + + +def fast_path_taken(t, keys, **aggs): + """Whether ``group_by(keys).agg(**aggs)`` is served by a fast path.""" + gb = t.group_by(keys) + specs = gb._normalize_aggs((), aggs) + return gb._mask_null_columns(specs), gb._try_fast_paths(specs, False) is not None + + +def test_null_free_mask_columns_keep_every_groupby_fast_path(): + """Decision 9 again: the deoptimization is scoped to columns with nulls.""" + t = table( + [(1, 2), (1, 3), (2, 4)], + k=blosc2.int64(null_storage="mask"), + v=blosc2.int64(null_storage="mask"), + ) + assert fast_path_taken(t, ["k"], total=("v", "sum")) == ([], True) + + +def test_a_mask_value_column_keeps_the_dense_single_key_path(): + """It is plain NumPy and already asks ``_null_mask``; it just needs the sidecar. + + Only the Cython kernels have to give up, because a kernel is handed a + ``skip_nan`` flag rather than a validity array. Keeping this path matters: + deferring a mask value column all the way to the generic hash-and-merge path + cost ~4.5x on a 2M-row sum. + """ + t = table( + [(1, 2), (1, 3), (2, 4)], + k=blosc2.int64(), + v=blosc2.int64(null_storage="mask"), + ) + t["v"][0] = None + assert fast_path_taken(t, ["k"], total=("v", "sum")) == (["v"], True) + + +def test_a_mask_key_column_leaves_the_fast_paths(): + """Its recoded chunk is a ``_CodedKeyChunk``, not an array of dense ints.""" + t = table( + [(1, 2), (1, 3), (2, 4)], + k=blosc2.int64(null_storage="mask"), + v=blosc2.int64(), + ) + t["k"][0] = None + assert fast_path_taken(t, ["k"], total=("v", "sum")) == (["k"], False) From 3c6d1f30ddd048651f8c01898e5e89c80c561847 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 16:32:11 +0200 Subject: [PATCH 09/47] Let columns migrate between null channels (mask-based-nulls 8) convert_nulls() moves a nullable column's nulls between an in-band sentinel and a .notnull sidecar, in either direction, for every V1 kind. It is never called implicitly: opening, copying and saving all preserve what a column already uses. Going back to a sentinel refuses what it cannot represent -- a column whose data already holds the proposed value, an int8 using all 256 -- and names the value in the way, rather than silently relabelling a real row as null. Every such reason is decided before a byte is written, so a refusal leaves the table untouched. The in-place ordering the plan recorded turns out to be wrong at its middle step: rewriting the null slots to the fill *before* the schema flips leaves a sentinel column reading a fill 0 as the value 0. Moving it last makes every intermediate state correct rather than merely recoverable -- sidecar, then schema, then fill, and the reverse for the other direction. Both are asserted. A dtype change is the one real limit: bool (uint8 <-> np.bool_) and a string too narrow for its sentinel need the stored array replaced, and no ordering of that write and the schema update survives a crash in between. Those refuse for a persistent inplace=True and point at inplace=False, which has no such window. Docs gain a "Where nulls are stored" section and a rewritten null-policy resolution order; info() now tags each column with where its nulls live. Three storage-independent bugs fixed on the way. copy() recorded its write watermark one below the exclusive bound every other writer means, so add_column() on a copied table backfilled one row short -- and raised outright for a variable-length column. _unflip_mask_bool_dtype keyed off the dtype rather than off whether the flip had happened, so a nullable uint8 ndarray column came back as bool_, every byte truncated to a flag. And copy() shares its schema object with the source, which conversion -- the one operation that mutates a spec in place -- was quietly reaching back through. Co-Authored-By: Claude Opus 5 --- RELEASE_NOTES.md | 61 +++ doc/reference/ctable.rst | 108 +++++- plans/mask-based-nulls.md | 56 ++- src/blosc2/ctable.py | 527 ++++++++++++++++++++++++-- src/blosc2/schema.py | 14 +- tests/ctable/test_null_migration.py | 555 ++++++++++++++++++++++++++++ 6 files changed, 1269 insertions(+), 52 deletions(-) create mode 100644 tests/ctable/test_null_migration.py diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ec1894b08..58e6831ea 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -4,6 +4,67 @@ XXX version-specific blurb XXX +### New features + +#### Mask-based nullable columns for CTable + +A nullable CTable column can now keep its nulls in a per-column **validity +sidecar** — Arrow's own model — instead of reserving a value from its own range. +Pass `null_storage="mask"` on any scalar spec: + +```python +blosc2.bool(null_storage="mask") # no reserved 255; dtype stays np.bool_ +blosc2.int8(null_storage="mask") # all 256 values usable, plus nulls +blosc2.utf8(null_storage="mask") # any string, including "" and "\x00" +blosc2.complex128(null_storage="mask") # nullable at all, for the first time +``` + +This is what makes nullability **lossless**: a sentinel steals a value from the +dtype, so a nullable `int8` could not hold `-128`, a free-text `utf8` column had +no safe sentinel at all, and Arrow columns whose type had no value to spare could +not be imported. `to_arrow(from_arrow(x))` now returns `x` for nullable `bool`, +full-range `int8`/`uint8`, `float64` containing `nan`/`±inf`/`-0.0` as values, +`utf8` containing `""` and `"__BLOSC2_NULL__"`, and `timestamp` with `int64.min` +as a value — none of which round-trip through a sentinel. + +Under mask storage `None` is how you write a null (`t.append((None,))`, +`t["price"][3] = None`), which a fixed-width sentinel column cannot accept at +all. `is_null()` is unchanged and remains the uniform API across every kind. + +Sentinel storage stays the **default** and is supported indefinitely; existing +tables open and behave exactly as before. `Column.null_storage` reports where a +column keeps its nulls and `info` tags each column (`int64 nullable[mask]`), so +`CTable.convert_nulls()` can move columns between the two in either direction — +never implicitly, and refusing rather than silently relabelling data when a +sentinel is unavailable. A table with a mask column records schema version 3, so +only such tables need a current reader. + +One deliberate semantic difference: in a mask column `NaN` is a **value**, +following Arrow, and only the sidecar marks a null. Sentinel float columns keep +NaN-as-null. See "Where nulls are stored" in the CTable reference. + +### Bug fixes + +- **`group_by` returned the wrong `min` for a `bool` value column.** The + per-group accumulator was seeded from the dtype's opposite identity, and `bool` + had none, so an all-`True` group reduced to `False`. Reachable with any plain + non-nullable bool column on the generic aggregation path. +- **Descending `sort_by` on a `bool` column raised**, and on a signed-integer + column holding its dtype's minimum (`-128` for `int8`) that row sorted as if it + were the largest. The descending key negated in the column's own dtype, where + `bool` has no unary minus and a narrow signed type wraps. +- **`add_column()` after `copy()` backfilled one row short**, and raised for a + variable-length column: the copy recorded its write watermark one below the + convention every other writer follows. +- **A string predicate over a nullable column returned its nulls as matches.** + `t.where("a > 10")` compared the stored sentinel, so any sentinel satisfying + the predicate (`null_value=999` against `> 10`) came back as a match. The + operator form (`t.where(t.a > 10)`) was always correct. Fixed for both storages. +- **A nullable `uint8` ndarray column came back as `bool`.** The `bool → uint8` + widening that sentinel storage needs was undone by dtype rather than by + whether it had been applied, so a column declared `uint8` was truncated to + flags. + ## Changes from 4.10.0 to 4.10.1 diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index acc764086..6ae8c87d5 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -118,13 +118,82 @@ the original unnamed root-list grouping, row groups, encoding choices, or file metadata exactly. +Where nulls are stored +---------------------- + +A nullable column keeps its nulls in one of two places, and +:attr:`Column.null_storage` says which:: + + t["price"].null_storage # 'sentinel', 'mask', 'code', 'native' or 'none' + t.info # tags every column: int64 nullable[mask] + +``"sentinel"`` + A value reserved from the column's own range — ``255`` for a nullable + ``bool`` (which is therefore physically ``uint8``), ``INT64_MIN`` for an + ``int64``, ``NaN`` for a float, the literal ``"__BLOSC2_NULL__"`` for a + string. Nothing extra is stored, and every reader understands it. + +``"mask"`` + A per-column ``.notnull`` validity array, one byte per row, ``True`` where + the value is present — Arrow's own model. Nullity lives outside the value + range, so the column keeps every value its dtype can hold, and a nullable + ``bool`` stays a real ``np.bool_``. + +The two other values are the only representation their kind has: ``"code"`` for +a dictionary column, which reserves ``-1``; ``"native"`` for the +variable-length container kinds, whose cells simply hold ``None``. + +Sentinel storage is the default and is supported indefinitely. Mask storage is +what makes nullability **lossless**, so it is worth asking for whenever data +comes from — or is going to — Arrow or Parquet: + +.. code-block:: python + + blosc2.bool(null_storage="mask") # no reserved 255; dtype stays np.bool_ + blosc2.int8(null_storage="mask") # all 256 values usable, plus nulls + blosc2.utf8(null_storage="mask") # any string, including "" and "\x00" + blosc2.complex128(null_storage="mask") # nullable at all, for the first time + +Under mask storage ``None`` is the way to write a null — ``t.append((None,))``, +``t["price"][3] = None`` — which a fixed-width sentinel column cannot accept at +all (there you write the sentinel yourself). Reads are unchanged: ``col[:]`` +returns values with a deterministic fill in the null slots, and +:meth:`Column.is_null` is what tells you which those are. + +One semantic difference is deliberate: in a **mask** column ``NaN`` is a +*value*, following Arrow, and only ``mask=False`` is a null. A sentinel float +column keeps NaN-as-null. So ``dropna``, ``group_by`` and ``min``/``max`` can +differ between the two for float columns holding a real NaN. + +Converting between them +~~~~~~~~~~~~~~~~~~~~~~~ + +Nothing migrates on its own: opening, copying and saving a table all preserve +each column's storage. :meth:`CTable.convert_nulls` is the only thing that +changes it:: + + lossless = t.convert_nulls() # every convertible column -> mask + lossless.copy(urlpath="out.b2d") # land it back on disk + t.convert_nulls("flag", to="mask", inplace=True) + +Going back to ``"sentinel"`` refuses what it cannot represent — a column whose +data already contains the proposed sentinel, or one with no value left to +reserve — and says which value is in the way, rather than silently relabelling +a real row as null. + +.. autosummary:: + + CTable.convert_nulls + +.. automethod:: CTable.convert_nulls + + Null policy ----------- -Nullable scalar CTable columns are represented with per-column sentinel values, -not native validity bitmaps. When CTable has to infer those sentinels, the -selection can be customized with :class:`NullPolicy` and scoped with -:func:`null_policy`:: +:class:`NullPolicy` decides what a bare ``nullable=True`` resolves to when the +schema does not say: which sentinel to reserve, or whether to use a mask at +all. Scope it with :func:`null_policy`:: policy = blosc2.NullPolicy( signed_int_strategy="max", @@ -148,10 +217,21 @@ The same policy is used by explicit nullable schema specs when no with blosc2.null_policy(policy): table = blosc2.CTable(Row) -Sentinels are resolved in this order: explicit ``null_value`` in the schema, -``NullPolicy.column_null_values`` for a matching column, then the type-wide -``NullPolicy`` default. Columns without ``nullable=True`` or an explicit -``null_value`` are not nullable. +Storage is resolved in this order, first match winning: an explicit +``null_storage=`` on the spec; an explicit ``null_value=`` (which *is* a request +for in-band storage); a ``NullPolicy.column_null_values`` entry for the column; +a type-wide ``NullPolicy`` sentinel field covering the column's kind; and +finally ``NullPolicy.null_storage``. Once sentinel storage is chosen, the +sentinel itself comes from ``column_null_values`` if listed and from the +type-wide default otherwise. + +Setting any type-wide sentinel field therefore *implies* sentinel storage for +the kinds it covers, so ``NullPolicy(float_value=-1.0)`` keeps working as it +always has. Passing ``null_storage="mask"`` alongside one is the one +combination that raises, because it contradicts itself. + +Columns without ``nullable=True``, an explicit ``null_value`` or an explicit +``null_storage`` are not nullable. .. autosummary:: @@ -198,9 +278,15 @@ expression: To exclude them, write the complementary comparison instead (``t.price <= 0``), which never matches nulls. +All of this applies to mask-storage columns unchanged: propagation reads an +opaque boolean "is null" predicate, which a sidecar supplies as directly as a +sentinel comparison does. + Kleene three-valued logic (where ``null > 0`` evaluates to null rather than -``False``) is intentionally out of scope — it needs a validity channel on -boolean intermediates, i.e. masks, which CTable does not use. +``False``) remains intentionally out of scope. Mask storage supplies the +validity channel it needs, but it also requires comparison *results* to carry +one, so that ``~`` can invert three states rather than two — a change to the +expression layer rather than to storage. Reductions on derived expressions skip nulls too: arithmetic involving a nullable column returns a ``NullableExpr`` — a thin wrapper that remembers @@ -737,10 +823,12 @@ Attributes Column.dtype Column.null_value + Column.null_storage Column.row_transformer .. autoproperty:: Column.dtype .. autoproperty:: Column.null_value +.. autoproperty:: Column.null_storage .. autoproperty:: Column.row_transformer diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 74e985989..acda5bf99 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,16 +1,19 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS — Phases 0–7 landed 2026-08-08.** Lossless Arrow/Parquet round-trip works -> for every V1 kind and sort/groupby/query now honour a sidecar, all **opt-in** via -> `null_storage="mask"`; next up is Phase 8 (migration + docs). Six premises were disproven during +> **Status: IN PROGRESS — Phases 0–8 landed 2026-08-08.** Lossless Arrow/Parquet round-trip works +> for every V1 kind, sort/groupby/query honour a sidecar, and `convert_nulls` migrates columns in +> either direction — all **opt-in** via `null_storage="mask"`; next up is Phase 9 (the default +> flip). Six premises were disproven during > implementation and are corrected in place, each in a blockquote beside the text it corrects: the > index path cannot be fixed by a null-aware expression (§Expression layer), the bool dtype-flip > cannot move out of `__init__` (§Schema layer), ndarray columns do not get lazy null propagation > for free (§Expression layer), the "free" summary min/max fast path is unsound (§Reductions), > `np.packbits` is not needed and avoiding it is safer (§Arrow/Parquet), and `.equals()` cannot -> express the round-trip contract (§Arrow/Parquet). Phase 7 added a seventh: **Phase 1 left the -> mask half of the query path undone** and nobody noticed until sort work went looking -> (§Expression layer, "Addendum 2"). Drafted 2026-08-08. +> express the round-trip contract (§Arrow/Parquet). Phase 7 added a seventh — **Phase 1 left the +> mask half of the query path undone**, and nobody noticed until sort work went looking +> (§Expression layer, "Addendum 2") — and Phase 8 an eighth: the in-place migration ordering +> recorded below is **wrong at its middle step**, and moving that step last makes every +> intermediate state correct rather than merely recoverable (§Migration). Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -777,6 +780,45 @@ the table intact. `inplace=False` (default, recommended) builds a new table via Detection is `Column.null_storage` plus an `info()` column — no separate report function. +> **Correction and as-built (2026-08-08).** The ordering above is **wrong at step 2, and the fix is +> to move it rather than to accept the window.** After (2) the table is *not* intact: the null slots +> now hold the fill, and a schema still saying `sentinel` reads a fill `0` as the value `0`. What +> shipped is **(1) sidecar, (2) schema, (3) fill** — and every intermediate state is then correct, +> not merely recoverable. A crash before (2) leaves an orphan `.notnull` key that a sentinel column +> never opens; a crash before (3) leaves a correct mask column whose null slots happen to still hold +> the old sentinel, which is unobservable through the `Column` API and which decision 5 explicitly +> excludes from the format contract. `to="sentinel"` runs the same argument backwards: sentinel into +> the null slots first (harmless while the sidecar is still authoritative), then the schema, then +> drop the sidecar. Both orderings are asserted in `test_null_migration.py`. +> +> Four further departures: +> +> - **A dtype change is refused for a persistent in-place conversion**, and this is the one real +> capability limit. `bool` (`uint8` ↔ `np.bool_`) and a `string`/`bytes` column too narrow for its +> sentinel need the stored array *replaced*, and there is no ordering of that write and the schema +> update a crash cannot land between. `inplace=False` has no such window — it builds a new +> table — so those columns raise, naming themselves and pointing at it. In-memory `inplace=True` is +> allowed: there is nothing to crash into. The check runs in `_convert_null_targets`, before any +> write, alongside the sentinel-availability checks, so a refusal never leaves litter. +> - **`copy()` shares its schema object with the source**, which conversion — the one operation that +> mutates a spec in place — cannot live with: a converted copy relabelled its *source's* columns +> too, leaving that table reporting a storage its data does not use. `_detach_schema()` deep-copies +> the specs first, and `_convert_nulls_inplace` always calls it, so `copy()` followed by an +> in-place conversion is safe as well. +> - **The all-elements ndarray rule needs no special handling.** `sentinel_mask(item_ndim=)` already +> implements it, and the sentinel direction writes the scalar sentinel across every element of a +> null row, which is the same rule read backwards. +> - **`Column.null_storage` and the `info()` rows already existed**, from Phases 0 and 4. What Phase +> 8 added is the *table-level* tag (`int64 nullable[mask]` in `info`'s per-column summary), which is +> what you actually read to decide whether a table needs converting. +> +> Also fixed here, storage-independent and pre-existing: `copy()` recorded its write watermark as +> `n - 1` where `_resolve_last_pos()` and every other writer mean an exclusive bound, so +> `add_column()` on a copied table backfilled one row short — and *raised* for a variable-length +> column. And `_unflip_mask_bool_dtype` keyed off the dtype rather than off whether the flip had +> happened, so a nullable **`uint8` ndarray** column under mask storage came back as `bool_`, every +> byte truncated to a flag; the specs now record `bool_widened_to_uint8`. + ## Verification **Differential oracle — the single highest-value test.** New @@ -846,7 +888,7 @@ default-created tables require them. | 5 | ✅ **Expressions + reductions.** `_ndarray_values_for_reduction`, argmin/argmax (both were reducing over the *fill*), `_reduction_null_mask` as the one storage-agnostic entry point. `_raw_null_pred`/`_lazy_nonnull_mask`/`_is_nullable_bool` needed nothing — Phases 0–4 had already made them storage-agnostic. **The ndarray-propagation gain is not real and was not done**, and the free summary fast path is unsound; both corrections are above. `tests/ctable/test_null_mask_expressions.py` (39 tests). | S (was M) | Low (was Med) | | 6 | ✅ **Arrow/Parquet.** Import + export for all V1 kinds; `arrow_slice(valid=)`; `null_storage=` on `from_arrow`/`from_parquet`; the "no sentinel available" import error now names the way out instead of being deleted (it still fires for sentinel storage, which still cannot represent those types). No `packbits` — pyarrow's own packing is borrowed instead. Ships **opt-in**; the default stays `"sentinel"`. `tests/ctable/test_null_mask_arrow.py` (42 tests). | M | Med | | 7 | ✅ **Sort + groupby.** `_build_lex_keys` (the indicator key is nulls-last *entirely*, not a refinement), `_sorted_positions_from_full_index` (3.0x for `U16`, 1.2x for `int64` — the I/O win is in bytes, not proportionally in time), `_utf8_rank_arrays(valid=)` plus a `null_aware` staleness rule no O(1) signal could replace, `_sorted_slice_positions` bails, groupby `_null_mask(valid=)` **plus `_CodedKeyChunk`**, which this section had not anticipated: a mask *key* column needs a reserved null code, not a threaded flag. **Plus the mask half of Phase 1**, which had never been done — `where()` leaked nulls on both the scan and the index (see §Expression layer, Addendum 2). Three pre-existing storage-independent bugs fixed on the way: descending sort of `bool` (raised) and of full-range signed ints (wrong order), and groupby `min`/`max` over `bool` (always `False`). `tests/ctable/test_null_mask_sort_groupby.py` (75 tests). | **L** (was M) | Med-High | -| 8 | **Migration + docs.** `convert_nulls`, `Column.null_storage`, `info()`, `doc/reference/ctable.rst` null-policy rewrite, release notes. | S–M | Low | +| 8 | ✅ **Migration + docs.** `convert_nulls` both directions for every V1 kind, refusing what a sentinel cannot represent; the crash ordering **corrected** (fill after the schema flip, not before) and asserted; a persistent in-place dtype change refused with a reason; `_detach_schema` so a converted copy stops relabelling its source; the table-level `info` null tag (`Column.null_storage` and the per-column `info` rows already existed). `doc/reference/ctable.rst` gains a "Where nulls are stored" section and a rewritten null-policy resolution order; release notes. Three storage-independent bugs fixed on the way: `copy()`'s off-by-one write watermark (which *raised* in `add_column`), and a nullable `uint8` ndarray column coming back as `bool_`. `tests/ctable/test_null_migration.py` (50 tests). | M (was S–M) | Low | | 9 | **Default flips to `"mask"`.** A one-line `NullPolicy` change plus release notes — lossless round-trip is why the default exists. Lands **no earlier than one release after Phase 6** so older readers in the wild already understand schema version 3. | S | Low | | 10 | **Index null-awareness remainder** *(independent)*. Mask-aware summary builder; `null_aware`/`null_order` descriptors; re-enable `_summary_minmax_source` for mask and sentinel columns alike. | **L** | High | diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 758ff5815..5631d509c 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -36,6 +36,7 @@ from blosc2.ctable_indexing import _CTableIndexingMixin from blosc2.ctable_nulls import ( NULL_MASK, + NULL_NONE, NULL_SENTINEL, NullChannel, fill_value_for, @@ -606,6 +607,15 @@ def __repr__(self) -> str: # --------------------------------------------------------------------------- +def _refresh_ndarray_dtype_fields(spec) -> None: + """Re-derive the dtype mirrors an :class:`NDArraySpec` caches beside ``dtype``.""" + spec.itemsize = spec.dtype.itemsize + spec.kind = spec.dtype.kind + spec.type = spec.dtype.type + spec.str = spec.dtype.str + spec.name = spec.dtype.name + + def _rank_index_row_lookup(values_path: str, positions_path: str, table, null_rank: int): """Build ``rows_for_ranks(lo, hi)`` over a rank index's sorted sidecars. @@ -2757,8 +2767,11 @@ def null_value(self): def null_storage(self) -> str: """How this column represents its nulls. - One of ``"none"``, ``"sentinel"``, ``"code"`` (dictionary) or - ``"native"`` (variable-length containers holding ``None`` cells). + One of ``"none"``, ``"mask"`` (a ``.notnull`` sidecar validity array, + Arrow's model), ``"sentinel"`` (a reserved in-band value), ``"code"`` + (dictionary) or ``"native"`` (variable-length containers holding ``None`` + cells). The first two are the ones :meth:`CTable.convert_nulls` moves + between; the last two are the only representation their kind has. """ return self._nulls.kind @@ -3757,9 +3770,10 @@ def info_items(self) -> list[tuple[str, object]]: ) else: col_meta = table._schema.columns_by_name.get(name) + spec = col_meta.spec if col_meta else None dtype_label = table._dtype_info_label( - getattr(table._cols[name], "dtype", None), col_meta.spec if col_meta else None - ) + getattr(table._cols[name], "dtype", None), spec + ) + table._null_info_tag(spec) cbytes = getattr(table._cols[name], "cbytes", None) if cbytes is not None: nbytes = getattr(table._cols[name], "nbytes", None) @@ -4781,18 +4795,29 @@ def _unflip_mask_bool_dtype(spec) -> None: *opening* a stored table -- which rebuilds specs without running this resolver -- brings a persisted uint8 column back as uint8. Once the policy has spoken for mask, the column is a real ``np.bool_`` again. + + Keyed off ``bool_widened_to_uint8``, not off the dtype: an **ndarray** + column may be ``uint8`` because that is what the user declared, and + turning that into ``np.bool_`` would truncate every byte to a flag. """ - if spec.dtype != np.dtype(np.uint8): + if not getattr(spec, "bool_widened_to_uint8", False): return - if isinstance(spec, b2_bool): - spec.dtype = np.dtype(np.bool_) - elif isinstance(spec, NDArraySpec): - spec.dtype = np.dtype(np.bool_) - spec.itemsize = spec.dtype.itemsize - spec.kind = spec.dtype.kind - spec.type = spec.dtype.type - spec.str = spec.dtype.str - spec.name = spec.dtype.name + spec.bool_widened_to_uint8 = False + spec.dtype = np.dtype(np.bool_) + if isinstance(spec, NDArraySpec): + _refresh_ndarray_dtype_fields(spec) + + @staticmethod + def _widen_bool_dtype_to_uint8(spec) -> None: + """Make room for the reserved ``255`` a sentinel-backed bool needs. + + The inverse of :meth:`_unflip_mask_bool_dtype`, and it records the flip + so that inverse knows it may be undone. + """ + spec.bool_widened_to_uint8 = True + spec.dtype = np.dtype(np.uint8) + if isinstance(spec, NDArraySpec): + _refresh_ndarray_dtype_fields(spec) @classmethod def _resolve_nullable_specs( @@ -4809,12 +4834,7 @@ def _resolve_nullable_specs( if isinstance(spec, NDArraySpec) and getattr(spec, "null_value", None) is not None: cls._validate_null_value_for_spec(col.name, spec, spec.null_value) if spec.dtype == np.dtype(np.bool_): - spec.dtype = np.dtype(np.uint8) - spec.itemsize = spec.dtype.itemsize - spec.kind = spec.dtype.kind - spec.type = spec.dtype.type - spec.str = spec.dtype.str - spec.name = spec.dtype.name + cls._widen_bool_dtype_to_uint8(spec) col.dtype = getattr(spec, "dtype", None) col.display_width = compute_display_width(spec) continue @@ -4851,15 +4871,10 @@ def _resolve_nullable_specs( elif isinstance(spec, b2_bytes): spec.max_length = max(spec.max_length, len(null_value), 1) spec.dtype = np.dtype(f"S{spec.max_length}") - elif isinstance(spec, b2_bool): - spec.dtype = np.dtype(np.uint8) - elif isinstance(spec, NDArraySpec) and spec.dtype == np.dtype(np.bool_): - spec.dtype = np.dtype(np.uint8) - spec.itemsize = spec.dtype.itemsize - spec.kind = spec.dtype.kind - spec.type = spec.dtype.type - spec.str = spec.dtype.str - spec.name = spec.dtype.name + elif isinstance(spec, b2_bool) or ( + isinstance(spec, NDArraySpec) and spec.dtype == np.dtype(np.bool_) + ): + cls._widen_bool_dtype_to_uint8(spec) col.dtype = getattr(spec, "dtype", None) col.display_width = compute_display_width(spec) @@ -6901,7 +6916,10 @@ def take(self, indices, /) -> CTable: result._valid_rows[:n] = True result._valid_rows[n:] = False result._n_rows = n - result._last_pos = n - 1 if n > 0 else None + # Exclusive bound, the convention _resolve_last_pos() documents and every + # other writer follows -- n - 1 here made add_column() on a copied table + # backfill one row short (and raise, for a varlen column). + result._last_pos = n if n > 0 else None return result def slice(self, start, stop=None, /, *, copy: bool = True) -> CTable: @@ -10030,6 +10048,434 @@ def _rename_stored_column(self, old: str, new: str): renamed_col = self._storage.open_varlen_scalar_column(new, old_compiled_col.spec) return renamed_col + # ------------------------------------------------------------------ + # Null-storage migration + # ------------------------------------------------------------------ + + #: Rows scanned per pass by :meth:`convert_nulls`. Bounds the transient + #: value and validity buffers without changing the result. + _CONVERT_NULLS_SPAN: ClassVar[int] = 1 << 20 + + def convert_nulls( + self, + columns: str | list[str] | None = None, + *, + to: str = "mask", + null_value: Any = None, + inplace: bool = False, + ) -> CTable: + """Convert nullable columns between sentinel and validity-mask storage. + + **Never called implicitly.** Opening, copying and saving a table all + preserve each column's existing null storage, so this is the only way a + column changes where it keeps its nulls. + + Parameters + ---------- + columns: + Column name, or list of names, to convert. ``None`` (default) + converts every column that *can* be converted and is not already in + the target storage. Naming a column that cannot be converted is an + error; an implicit sweep skips it silently. + to: + ``"mask"`` (default) moves nullity into a ``.notnull`` sidecar -- + Arrow's model, and the only one that is lossless for every type. + ``"sentinel"`` moves it back in band, and **refuses what it cannot + represent**: a column whose data already contains the proposed + sentinel, or one with no value left to reserve. + null_value: + The sentinel to reserve, for ``to="sentinel"``. Only accepted when + converting exactly one column, since a single value cannot be right + for several kinds at once; otherwise the active + :class:`NullPolicy`'s type-wide default is used. + inplace: + If ``True``, rewrite this table and return ``self``. If ``False`` + (default, and recommended) return a new **in-memory** table, leaving + this one untouched -- pass the result to ``copy(urlpath=...)`` to + land it back on disk. + + ``inplace=True`` on a *persistent* table cannot change a column's + dtype, which rules out ``bool`` (``uint8`` <-> ``np.bool_``) and a + ``string``/``bytes`` column too narrow for the sentinel it is being + given. Replacing a stored array means there is a moment when the + values and the schema disagree, and no ordering of the two survives + a crash in between; ``inplace=False`` has no such moment because it + builds a new table. Those columns raise, naming themselves. + + Notes + ----- + Everything else *is* crash-safe, and the ordering is the argument: + + ``to="mask"`` writes the complete sidecar first, then flips the schema, + then normalizes the null slots to the fill. A crash before the flip + leaves an orphan ``.notnull`` key that a sentinel column never reads; a + crash after it leaves a correct mask column whose null slots happen to + still hold the old sentinel -- which is unobservable except as raw + values, and the fill is explicitly not part of the format contract. + + ``to="sentinel"`` runs the same argument backwards: it writes the + sentinel into the null slots *first* (harmless while the sidecar is + still authoritative), then flips the schema, then drops the sidecar. + + A **null-free** column converts as a pure schema update in either + direction: nothing is scanned twice, and to="mask" writes no sidecar at + all, since an absent one already means "every row is valid". + + ``string``/``bytes`` columns keep the ``max_length`` a sentinel forced on + them -- shrinking it is a dtype change, so use ``copy()`` to reclaim the + width. Dictionary columns (reserved code) and the variable-length + container kinds (native ``None`` cells) have only one representation and + are never converted. + + Examples + -------- + >>> lossless = t.convert_nulls() # doctest: +SKIP + >>> lossless.copy(urlpath="out.b2d") # doctest: +SKIP + >>> t.convert_nulls("flag", to="mask", inplace=True) # doctest: +SKIP + """ + if to not in (NULL_MASK, NULL_SENTINEL): + raise ValueError(f"to must be 'mask' or 'sentinel', got {to!r}") + if null_value is not None and to != NULL_SENTINEL: + raise ValueError("null_value only applies to to='sentinel'") + if isinstance(columns, str): + columns = [columns] + # A dtype change means replacing an array, which only an in-memory table + # can do safely; inplace=False lands there by construction, since it + # converts the copy. + targets = self._convert_null_targets( + columns, + to, + null_value, + allow_recast=not (inplace and isinstance(self._storage, FileTableStorage)), + ) + + if not inplace: + # copy() preserves each column's null storage (decision 7), so the + # conversion happens exactly once, on a table nobody else holds -- + # and in memory, where replacing an array costs nothing. + result = self.copy() + result._convert_nulls_inplace(targets, to) + return result + if self._read_only: + raise ValueError("Table is read-only (opened with mode='r').") + if self.base is not None: + raise ValueError( + "Cannot convert a view's null storage inplace (it shares the base table's " + "columns). Use convert_nulls(inplace=False) to get a converted copy." + ) + self._convert_nulls_inplace(targets, to) + return self + + def _convert_null_targets( + self, columns: list[str] | None, to: str, null_value: Any, *, allow_recast: bool + ) -> list[tuple[str, Any]]: + """Resolve the columns to convert into ``[(name, sentinel)]``. + + *sentinel* is the value to reserve (``to="sentinel"``) or ``None``. + Everything that can make a conversion impossible is decided here, before + a single byte is written -- an unavailable sentinel, one the data already + uses, a dtype change a persistent table cannot make -- so a refusal + leaves the table exactly as it was rather than half converted. + """ + if columns is None: + names = [ + col.name + for col in self._schema.columns + if kind_of_spec(col.spec) in (NULL_MASK, NULL_SENTINEL) and kind_of_spec(col.spec) != to + ] + else: + names = [] + for name in columns: + col_info = self._schema.columns_by_name.get(name) + if col_info is None: + raise KeyError(f"Column {name!r} not found.") + kind = kind_of_spec(col_info.spec) + if kind == NULL_NONE: + raise ValueError(f"Column {name!r} is not nullable, so it has no nulls to convert.") + if kind not in (NULL_MASK, NULL_SENTINEL): + raise ValueError( + f"Column {name!r} stores its nulls as {kind!r}, which is the only " + f"representation its kind has -- a dictionary column reserves a code and " + f"the variable-length kinds hold native None. There is nothing to convert." + ) + if kind != to: + names.append(name) # already converted: skip, so this stays idempotent + if null_value is not None and len(names) != 1: + raise ValueError( + f"null_value applies to a single column, but {len(names)} would be converted; " + "name one column, or drop null_value to use the NullPolicy default for each kind." + ) + if to == NULL_MASK: + targets = [(name, None) for name in names] + else: + targets = [(name, self._sentinel_for_conversion(name, null_value)) for name in names] + if not allow_recast: + for name, sentinel in targets: + spec = self._schema.columns_by_name[name].spec + if self._convert_changes_dtype(spec, to, sentinel): + raise ValueError( + f"Converting column {name!r} changes its physical dtype (a nullable bool is " + f"{'uint8 under sentinel storage' if to == NULL_SENTINEL else 'np.bool_ under mask storage'}" + f"; a string or bytes column too narrow for its sentinel has to grow), which " + f"means replacing the stored array -- and there is no ordering of that write " + f"and the schema update that a crash cannot land between. Use " + f"convert_nulls(inplace=False), then copy(urlpath=...) to store the result." + ) + return targets + + def _sentinel_for_conversion(self, name: str, null_value: Any): + """Pick and vet the sentinel column *name* will reserve. + + Rejects, before any write, the two ways a sentinel can be unavailable: + a kind with no value to spare at all, and a value the column's own data + already uses -- which would silently relabel real rows as null, the very + loss mask storage exists to avoid. + """ + spec = self._schema.columns_by_name[name].spec + if not getattr(spec, "supports_sentinel", True): + raise ValueError( + f"Column {name!r} has dtype {spec.dtype}, for which no value can be reserved as a " + f"null sentinel. It can only use mask storage." + ) + if null_value is None: + null_value = self._policy_null_value_for_spec(spec, get_null_policy()) + if null_value is None: + raise ValueError( + f"No null policy sentinel is available for column {name!r}; pass " + f"null_value= explicitly, or keep the column on mask storage." + ) + self._validate_null_value_for_spec(name, spec, null_value) + col = self[name] + clash = self._convert_sentinel_clash(col, spec, null_value) + if clash is not None: + raise ValueError( + f"Column {name!r} already contains {null_value!r} at row {clash} as a real value, " + f"so reserving it as the null sentinel would relabel that row as null. Pass a " + f"different null_value=, or keep the column on mask storage." + ) + return null_value + + def _convert_sentinel_clash(self, col: Column, spec, null_value) -> int | None: + """The first non-null row already holding *null_value*, or ``None``. + + A NaN sentinel never clashes: under mask storage NaN is a value, but it + is one the sentinel model has always spelled "null", so folding the two + together is the documented semantic change rather than data loss. + """ + if is_nan_sentinel(null_value): + return None + valid_arr = self._null_mask(col._col_name) + item_ndim = col.item_ndim if col.is_ndarray else 0 + for start, stop, raw in self._iter_convert_spans(col._col_name): + hit = sentinel_mask(raw, null_value, item_ndim=item_ndim) + if valid_arr is not None: + hit &= np.asarray(valid_arr[start:stop], dtype=bool) + if hit.any(): + return start + int(np.flatnonzero(hit)[0]) + return None + + def _iter_convert_spans(self, name: str): + """Yield ``(start, stop, values)`` over column *name*'s written slots. + + Physical, and bounded to the write watermark rather than the capacity: + the padding past it holds no row, and a utf8 column is not even that + long. + """ + arr = self._cols[name] + n_phys = min(self._resolve_last_pos(), len(arr)) + span = self._CONVERT_NULLS_SPAN + for start in range(0, n_phys, span): + stop = min(start + span, n_phys) + yield start, stop, np.asarray(arr[start:stop]) + + def _detach_schema(self) -> None: + """Give this table its own copy of every column spec. + + ``_empty_copy`` hands the copy the *same* :class:`CompiledSchema` object, + which is right for everything that only reads it -- and wrong for the one + thing that does not. Conversion rewrites specs **in place**, and without + this a converted copy would reach back and relabel its source's columns + too, leaving that table reporting a storage its data does not use. + """ + columns = [dataclasses.replace(c, spec=copy.deepcopy(c.spec)) for c in self._schema.columns] + self._schema = CompiledSchema( + row_cls=self._schema.row_cls, + columns=columns, + columns_by_name={c.name: c for c in columns}, + ) + + def _convert_nulls_inplace(self, targets: list[tuple[str, Any]], to: str) -> None: + """Rewrite each target column's null channel, one column at a time. + + Per column rather than per phase, so an interrupted conversion leaves + every other column exactly as it was. + """ + self._detach_schema() + for name, sentinel in targets: + if to == NULL_MASK: + self._convert_column_to_mask(name) + else: + self._convert_column_to_sentinel(name, sentinel) + + def _convert_column_to_mask(self, name: str) -> None: + col_info = self._schema.columns_by_name[name] + spec = col_info.spec + sentinel = spec.null_value + item_ndim = self[name].item_ndim if self[name].is_ndarray else 0 + + # 1. The sidecar, complete, before the schema mentions it. + mask_arr = None + for start, stop, raw in self._iter_convert_spans(name): + null = sentinel_mask(raw, sentinel, item_ndim=item_ndim) + if not null.any(): + continue + if mask_arr is None: + mask_arr = self._ensure_null_mask(name) + # create_null_mask zero-fills, and zero is *invalid*; start from + # all-valid and punch the nulls out below. + mask_arr[:] = True + mask_arr[start:stop] = ~null + + # 2. The schema. A column with no null needs nothing else: an absent + # sidecar already says every row is valid (decision 9). + self._retype_converted_column(name, to=NULL_MASK, null_value=None) + if mask_arr is None: + return + + # 3. Normalize the null slots to the fill. Purely cosmetic -- what sits + # under valid=False is unobservable through the Column API -- so it + # runs last, where an interruption costs nothing. A bool column + # *needs* it: step 2 recast uint8 to np.bool_, turning the old 255 + # sentinel into True. + self._write_at_invalid(name, mask_arr, self[name]._nulls.fill_value) + + def _write_at_invalid(self, name: str, mask_arr, value) -> None: + """Overwrite the values under ``mask_arr == False`` with *value*.""" + col = self[name] + arr = self._cols[name] + n_phys = min(self._resolve_last_pos(), len(arr)) + span = self._CONVERT_NULLS_SPAN + for start in range(0, n_phys, span): + stop = min(start + span, n_phys) + invalid = ~np.asarray(mask_arr[start:stop], dtype=bool) + if not invalid.any(): + continue + positions = np.flatnonzero(invalid) + start + if col.is_utf8: + # A UTF8Array assigns one row at a time; the null rows are a + # small subset by construction. + for pos in positions: + arr[int(pos)] = value + elif isinstance(value, np.ndarray): + # An ndarray column's fill is a whole *item*; the fancy-index + # setitem wants one item per selected row rather than something + # to broadcast across them. + arr[positions] = np.broadcast_to(value, (len(positions), *value.shape)) + else: + arr[positions] = value + + def _convert_column_to_sentinel(self, name: str, sentinel) -> None: + spec = self._schema.columns_by_name[name].spec + mask_arr = self._null_mask(name) + # A bool column's 255 does not fit np.bool_, and a widened string's + # sentinel does not fit the old itemsize, so those have to be recast + # before the sentinel can be written at all. Both are in-memory-only + # (see _recast_converted_column), where the ordering below buys nothing. + recast_first = self._convert_changes_dtype(spec, NULL_SENTINEL, sentinel) + + if recast_first: + self._retype_converted_column(name, to=NULL_SENTINEL, null_value=sentinel) + if mask_arr is not None: + # The sentinel goes into the null slots while the sidecar is still + # what says which those are, so this write cannot lose anything. + self._write_at_invalid(name, mask_arr, sentinel) + if not recast_first: + # The schema last, which is what makes those slots mean "null". + self._retype_converted_column(name, to=NULL_SENTINEL, null_value=sentinel) + if mask_arr is not None: + # Now dead weight: a sentinel column never reads a sidecar. + self._drop_null_mask(name) + + @staticmethod + def _convert_changes_dtype(spec, to: str, null_value) -> bool: + """Whether this conversion also changes the column's physical dtype. + + Only three ways it can: a bool column loses or regains the ``uint8`` + widening, and a ``string``/``bytes`` column too narrow for its new + sentinel has to grow. + """ + if to == NULL_MASK: + return getattr(spec, "bool_widened_to_uint8", False) is True + if isinstance(spec, (string, b2_bytes)): + return len(null_value) > spec.max_length + if isinstance(spec, b2_bool): + return spec.dtype != np.dtype(np.uint8) + return isinstance(spec, NDArraySpec) and spec.dtype == np.dtype(np.bool_) + + def _retype_converted_column(self, name: str, *, to: str, null_value) -> None: + """Rewrite column *name*'s spec for its new null channel, and persist it. + + The dtype consequences are the same ones ``_resolve_nullable_specs`` + applies at creation, run in the other direction: mask storage undoes the + ``bool -> uint8`` flip, and sentinel storage reapplies it and widens a + ``string``/``bytes`` column that cannot hold its sentinel. A dtype + change means a new array, which is why it is refused for a persistent + in-place conversion. + """ + col_info = self._schema.columns_by_name[name] + spec = col_info.spec + before = spec.dtype + spec.nullable = True + spec.null_storage = to + spec.null_value = null_value + if to == NULL_MASK: + self._unflip_mask_bool_dtype(spec) + elif isinstance(spec, string): + spec.max_length = max(spec.max_length, len(null_value), 1) + spec.dtype = np.dtype(f"U{spec.max_length}") + elif isinstance(spec, b2_bytes): + spec.max_length = max(spec.max_length, len(null_value), 1) + spec.dtype = np.dtype(f"S{spec.max_length}") + elif isinstance(spec, b2_bool) or ( + isinstance(spec, NDArraySpec) and spec.dtype == np.dtype(np.bool_) + ): + self._widen_bool_dtype_to_uint8(spec) + if spec.dtype != before: + self._recast_converted_column(name, col_info, before) + col_info.dtype = getattr(spec, "dtype", None) + col_info.display_width = compute_display_width(spec) + self._col_widths[name] = max(len(name), col_info.display_width) + if isinstance(self._storage, FileTableStorage): + self._storage.save_schema(self._schema_dict_with_computed()) + + def _recast_converted_column(self, name: str, col_info, before: np.dtype) -> None: + """Replace column *name*'s array so it matches its new dtype. + + In-memory only, and :meth:`_convert_null_targets` has already refused the + persistent case: there the values and the schema would disagree for as + long as the write takes, and no ordering of the two survives a crash in + between. Here there is no crash window to protect, so the values are + simply cast across. + """ + spec = col_info.spec + old = self._cols[name] + shape = self._column_physical_shape(col_info, len(self._valid_rows)) + new = self._storage.create_column( + name, + dtype=spec.dtype, + shape=shape, + chunks=old.chunks, + blocks=old.blocks, + cparams=None, + dparams=None, + ) + n_phys = min(self._resolve_last_pos(), len(old)) + span = self._CONVERT_NULLS_SPAN + for start in range(0, n_phys, span): + stop = min(start + span, n_phys) + new[start:stop] = np.asarray(old[start:stop]).astype(spec.dtype, copy=False) + self._cols[name] = new + # ------------------------------------------------------------------ # Computed / virtual columns # ------------------------------------------------------------------ @@ -12879,7 +13325,10 @@ def copy( # noqa: C901 if compact: result._valid_rows[:n] = True result._n_rows = n - result._last_pos = n - 1 if n > 0 else None + # Exclusive bound, the convention _resolve_last_pos() documents and + # every other writer follows -- n - 1 here made add_column() on a + # copied table backfill one row short (and raise, for a varlen one). + result._last_pos = n if n > 0 else None else: result._valid_rows[:n] = valid_np[:n] result._n_rows = n_live @@ -13141,9 +13590,10 @@ def info_items(self) -> list[tuple[str, object]]: ) else: col_meta = self._schema.columns_by_name.get(name) + spec = col_meta.spec if col_meta else None dtype_label = self._dtype_info_label( - getattr(self._cols[name], "dtype", None), col_meta.spec if col_meta else None - ) + getattr(self._cols[name], "dtype", None), spec + ) + self._null_info_tag(spec) cbytes = getattr(self._cols[name], "cbytes", None) if cbytes is not None: nbytes = getattr(self._cols[name], "nbytes", None) @@ -13209,6 +13659,17 @@ def _valid_rows_info_label(self) -> str: detail += f", cratio: {nbytes / cbytes:.2f}x" return f"{dtype_label} ({detail})" + @staticmethod + def _null_info_tag(spec: SchemaSpec | None) -> str: + """``" nullable[mask]"`` and friends, or ``""`` for a non-nullable column. + + Appended to a column's dtype in :attr:`info`'s per-column summary, which + is where a whole table's null storage becomes visible at a glance -- what + you read before deciding whether to run :meth:`convert_nulls`. + """ + kind = kind_of_spec(spec) + return "" if kind == NULL_NONE else f" nullable[{kind}]" + @staticmethod def _dtype_info_label(dtype: np.dtype | None, spec: SchemaSpec | None = None) -> str: """Return a compact dtype label for info reports.""" diff --git a/src/blosc2/schema.py b/src/blosc2/schema.py index ee86d0ea5..a95152e6e 100644 --- a/src/blosc2/schema.py +++ b/src/blosc2/schema.py @@ -64,6 +64,13 @@ class _NullableSpecMixin: supports_sentinel = True + #: True once ``__init__`` has physically widened a boolean column to + #: ``uint8`` to make room for the reserved ``255``. Only that flip may be + #: undone (``CTable._unflip_mask_bool_dtype``): a column whose *declared* + #: dtype is ``uint8`` is carrying real byte values, and turning it into + #: ``np.bool_`` would truncate every one of them to a flag. + bool_widened_to_uint8 = False + def _init_nulls(self, *, nullable, null_value, null_storage) -> None: if null_storage is not None and null_storage not in _EXPLICIT_NULL_STORAGES: raise ValueError( @@ -431,7 +438,8 @@ def __init__(self, *, nullable: bool = False, null_value=None, null_storage: str # persisted as uint8 has to come back as uint8 from its metadata # alone. When storage is still unresolved (plain ``nullable=True``, # policy decides later) the resolver corrects this both ways. - self.dtype = np.dtype(np.uint8) if self.nullable and not self.uses_mask else np.dtype(np.bool_) + self.bool_widened_to_uint8 = _builtin_bool(self.nullable and not self.uses_mask) + self.dtype = np.dtype(np.uint8) if self.bool_widened_to_uint8 else np.dtype(np.bool_) def to_pydantic_kwargs(self) -> dict[str, Any]: return {} @@ -954,7 +962,9 @@ def __init__( if self.nullable and not self.uses_mask and self.dtype == np.dtype(np.bool_): # Same reasoning as bool: opening a stored table rebuilds the spec # without running the resolver, so the uint8 flip has to survive - # from metadata alone. + # from metadata alone. Recorded, because a *declared* uint8 ndarray + # column holds real byte values and must never be unflipped. + self.bool_widened_to_uint8 = True self.dtype = np.dtype(np.uint8) self.itemsize = self.dtype.itemsize self.kind = self.dtype.kind diff --git a/tests/ctable/test_null_migration.py b/tests/ctable/test_null_migration.py new file mode 100644 index 000000000..6f254cbed --- /dev/null +++ b/tests/ctable/test_null_migration.py @@ -0,0 +1,555 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""``convert_nulls``: moving a column's nulls between channels (Phase 8). + +Two guarantees frame everything here. **Nothing auto-migrates** -- opening, +copying and saving a table all preserve each column's ``null_storage``, so this +is the only thing that changes it. And a conversion either happens or does not: +every reason one can fail is decided before a byte is written, so a refusal +leaves the table exactly as it was rather than half converted. + +The crash-safety ordering is asserted directly, because it is the whole argument +for why an in-place conversion is allowed at all: each intermediate state on disk +has to read correctly under *some* schema, and the schema is only ever the last +thing to move. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 + + +def annotation_for(spec): + if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)): + return object + return spec.python_type + + +def table(rows, capacity=64, urlpath=None, **cols): + Row = dataclasses.make_dataclass( + "MigrateRow", [(n, annotation_for(s), blosc2.field(s)) for n, s in cols.items()] + ) + kwargs = {"urlpath": str(urlpath), "mode": "w"} if urlpath is not None else {} + t = blosc2.CTable(Row, expected_size=max(capacity, len(rows)), **kwargs) + if rows: + t.extend(rows) + return t + + +def one_col(values, spec, capacity=64, urlpath=None): + return table([(v,) for v in values], capacity=capacity, urlpath=urlpath, a=spec) + + +def as_list(col): + """A column's live values with nulls spelled ``None``.""" + null = col.is_null() + return [None if null[i] else _scalar(col[i]) for i in range(len(col))] + + +def _scalar(value): + return value.item() if hasattr(value, "item") else value + + +#: Every V1 kind, with a value it can hold and the sentinel its policy picks. +V1_KINDS = [ + ("int64", blosc2.int64, {}, [5, None, 1]), + ("int8", blosc2.int8, {}, [5, None, 1]), + ("uint8", blosc2.uint8, {}, [5, None, 1]), + ("float64", blosc2.float64, {}, [1.5, None, 2.5]), + ("bool", blosc2.bool, {}, [True, None, False]), + ("string", blosc2.string, {"max_length": 4}, ["ab", None, "cd"]), + ("bytes", blosc2.bytes, {"max_length": 4}, [b"ab", None, b"cd"]), + ("utf8", blosc2.utf8, {}, ["ab", None, "cdefgh"]), + ( + "timestamp", + blosc2.timestamp, + {}, + [np.datetime64("2020-01-01"), None, np.datetime64("2021-06-01")], + ), +] + + +# --------------------------------------------------------------------------- +# Round trip, both directions, every kind +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize(("label", "factory", "kw", "values"), V1_KINDS) +def test_mask_to_sentinel_to_mask_preserves_the_data(label, factory, kw, values): + """Both directions, back to back, for each kind that has two channels. + + The sentinel leg is the lossy one by construction -- it has to steal a value + from the range -- so what this pins is that the *data present here* survives, + not that any data would. ``test_to_sentinel_refuses_*`` covers the rest. + """ + mask = one_col(values, factory(null_storage="mask", **kw)) + assert mask["a"].null_storage == "mask" + + sent = mask.convert_nulls("a", to="sentinel") + assert sent["a"].null_storage == "sentinel" + assert sent["a"].null_value is not None + assert as_list(sent["a"]) == as_list(mask["a"]) + + back = sent.convert_nulls("a", to="mask") + assert back["a"].null_storage == "mask" + assert as_list(back["a"]) == as_list(mask["a"]) + assert back["a"].dtype == mask["a"].dtype or label in ("string", "bytes") + + +def test_conversion_leaves_the_source_untouched(): + t = one_col([5, None, 1], blosc2.int64(null_storage="mask")) + converted = t.convert_nulls("a", to="sentinel") + assert t["a"].null_storage == "mask" + assert t["a"].null_value is None + assert converted["a"].null_storage == "sentinel" + + +def test_implicit_sweep_converts_every_convertible_column(): + t = table( + [(5, "ab", 1), (None, None, 2)], + k=blosc2.int64(null_storage="mask"), + s=blosc2.string(max_length=2, null_storage="mask"), + plain=blosc2.int64(), + ) + sent = t.convert_nulls(to="sentinel") + assert sent["k"].null_storage == "sentinel" + assert sent["s"].null_storage == "sentinel" + assert sent["plain"].null_storage == "none" # not nullable: nothing to convert + + +def test_implicit_sweep_skips_what_it_cannot_convert(): + """A dictionary column has one representation, so a sweep passes over it.""" + t = table( + [("x", 5), (None, None)], + d=blosc2.dictionary(), + k=blosc2.int64(null_storage="mask"), + ) + sent = t.convert_nulls(to="sentinel") + assert sent["d"].null_storage == "code" + assert sent["k"].null_storage == "sentinel" + + +def test_converting_is_idempotent(): + t = one_col([5, None, 1], blosc2.int64(null_storage="mask")) + once = t.convert_nulls("a", to="mask") + twice = once.convert_nulls("a", to="mask") + assert twice["a"].null_storage == "mask" + assert as_list(twice["a"]) == [5, None, 1] + + +# --------------------------------------------------------------------------- +# A null-free column converts as a pure schema update +# --------------------------------------------------------------------------- + + +def test_null_free_sentinel_column_writes_no_sidecar(): + """Decision 9 reaching migration: an absent sidecar already says all-valid.""" + t = one_col([1, 2, 3], blosc2.int64(nullable=True, null_value=-1)) + converted = t.convert_nulls("a", to="mask") + assert converted["a"].null_storage == "mask" + assert converted._null_mask("a") is None + assert converted["a"].is_null().tolist() == [False, False, False] + assert as_list(converted["a"]) == [1, 2, 3] + + +def test_null_free_mask_column_needs_no_value_rewrite(): + t = one_col([1, 2, 3], blosc2.int64(null_storage="mask")) + assert t._null_mask("a") is None + converted = t.convert_nulls("a", to="sentinel") + assert converted["a"].null_storage == "sentinel" + assert as_list(converted["a"]) == [1, 2, 3] + + +def test_converting_drops_the_sidecar_on_the_way_to_sentinel(): + t = one_col([5, None, 1], blosc2.int64(null_storage="mask")) + converted = t.convert_nulls("a", to="sentinel") + assert converted._null_mask("a") is None + assert converted["a"].is_null().tolist() == [False, True, False] + + +# --------------------------------------------------------------------------- +# to="sentinel" refuses what it cannot represent +# --------------------------------------------------------------------------- + + +def test_to_sentinel_refuses_a_full_range_int8(): + """The case that motivated the design, refused rather than silently lossy. + + ``int8`` using all 256 values has nothing left to reserve, which is exactly + why mask storage exists; converting back would relabel a real ``-128``. + """ + t = one_col([*range(-128, 128), None], blosc2.int8(null_storage="mask"), capacity=300) + with pytest.raises(ValueError, match="already contains -128"): + t.convert_nulls("a", to="sentinel") + + +def test_to_sentinel_refuses_utf8_holding_the_sentinel_string(): + t = one_col(["__BLOSC2_NULL__", None], blosc2.utf8(null_storage="mask")) + with pytest.raises(ValueError, match="already contains"): + t.convert_nulls("a", to="sentinel") + + +def test_to_sentinel_accepts_a_different_sentinel_instead(): + """The refusal names the offending value, and a free one is accepted.""" + t = one_col(["__BLOSC2_NULL__", None], blosc2.utf8(null_storage="mask")) + converted = t.convert_nulls("a", to="sentinel", null_value="\x00\x01") + assert converted["a"].null_value == "\x00\x01" + assert as_list(converted["a"]) == ["__BLOSC2_NULL__", None] + + +def test_to_sentinel_refuses_complex(): + """No value can be spared from the complex plane, so it is mask or nothing.""" + t = one_col([1 + 2j, None], blosc2.complex128(null_storage="mask")) + with pytest.raises(ValueError, match="no value can be reserved"): + t.convert_nulls("a", to="sentinel") + + +def test_a_nan_already_present_does_not_block_the_nan_sentinel(): + """Folding NaN into "null" is the documented semantic change, not data loss. + + A sentinel float column has always spelled its nulls ``NaN``; converting a + mask column that contains a real NaN therefore *changes what that row means*, + which decision 6 covers, rather than losing a value the way an integer + sentinel collision would. + """ + t = one_col([1.0, float("nan"), None], blosc2.float64(null_storage="mask")) + converted = t.convert_nulls("a", to="sentinel") + assert converted["a"].is_null().tolist() == [False, True, True] + + +def test_the_refusal_leaves_the_column_alone(): + t = one_col([*range(-128, 128), None], blosc2.int8(null_storage="mask"), capacity=300) + with pytest.raises(ValueError): + t.convert_nulls("a", to="sentinel") + assert t["a"].null_storage == "mask" + assert t["a"].null_value is None + assert t["a"][:-1].tolist() == list(range(-128, 128)) + + +# --------------------------------------------------------------------------- +# Naming a column that cannot convert is an error +# --------------------------------------------------------------------------- + + +def test_naming_a_dictionary_column_raises(): + t = one_col(["x", None], blosc2.dictionary()) + with pytest.raises(ValueError, match="only representation its kind has"): + t.convert_nulls("a", to="mask") + + +def test_naming_a_non_nullable_column_raises(): + t = one_col([1, 2], blosc2.int64()) + with pytest.raises(ValueError, match="not nullable"): + t.convert_nulls("a", to="mask") + + +def test_naming_an_unknown_column_raises(): + t = one_col([1, 2], blosc2.int64()) + with pytest.raises(KeyError, match="not found"): + t.convert_nulls("nope") + + +def test_bad_to_value_raises(): + t = one_col([1, 2], blosc2.int64()) + with pytest.raises(ValueError, match="must be 'mask' or 'sentinel'"): + t.convert_nulls(to="bitmap") + + +def test_null_value_with_to_mask_raises(): + t = one_col([1, 2], blosc2.int64()) + with pytest.raises(ValueError, match="only applies to to='sentinel'"): + t.convert_nulls(to="mask", null_value=7) + + +def test_null_value_across_several_columns_raises(): + """One value cannot be right for several kinds, so it has to name a column.""" + t = table( + [(5, "ab"), (None, None)], + k=blosc2.int64(null_storage="mask"), + s=blosc2.string(max_length=2, null_storage="mask"), + ) + with pytest.raises(ValueError, match="applies to a single column"): + t.convert_nulls(to="sentinel", null_value=-1) + + +# --------------------------------------------------------------------------- +# ndarray columns +# --------------------------------------------------------------------------- + + +def ndarray_table(dtype, item_shape, rows, **spec_kw): + spec = blosc2.ndarray(dtype=dtype, item_shape=item_shape, **spec_kw) + Row = dataclasses.make_dataclass("NdRow", [("v", object, blosc2.field(spec))]) + t = blosc2.CTable(Row, expected_size=16) + t.extend([(r,) for r in rows]) + return t + + +def test_ndarray_column_converts_both_ways(): + t = ndarray_table(np.int32, (3,), [[1, 2, 3], None, [4, 5, 6]], null_storage="mask") + sent = t.convert_nulls("v", to="sentinel") + assert sent["v"].is_null().tolist() == [False, True, False] + # The old rule: a row is null only when *every* element holds the sentinel. + assert sent["v"][:][1].tolist() == [np.iinfo(np.int32).min] * 3 + back = sent.convert_nulls("v", to="mask") + assert back["v"].is_null().tolist() == [False, True, False] + assert back["v"][:][0].tolist() == [1, 2, 3] + + +def test_bool_ndarray_column_changes_dtype_both_ways(): + t = ndarray_table(np.bool_, (2,), [[True, False], None], null_storage="mask") + assert t["v"].dtype == np.dtype(np.bool_) + sent = t.convert_nulls("v", to="sentinel") + assert sent["v"].dtype == np.dtype(np.uint8) + assert sent["v"][:].tolist() == [[1, 0], [255, 255]] + back = sent.convert_nulls("v", to="mask") + assert back["v"].dtype == np.dtype(np.bool_) + assert back["v"].is_null().tolist() == [False, True] + assert back["v"][:][0].tolist() == [True, False] + + +def test_a_declared_uint8_ndarray_column_is_not_mistaken_for_a_widened_bool(): + """The unflip must undo only the flip it made. + + ``bool`` columns are physically ``uint8`` under sentinel storage, and mask + storage undoes that -- but a column whose *declared* dtype is ``uint8`` holds + real byte values, and turning it into ``np.bool_`` would truncate every one + of them to a flag. Keyed off a recorded flag rather than off the dtype. + """ + spec = blosc2.ndarray(dtype=np.uint8, item_shape=(2,), nullable=True, null_storage="mask") + Row = dataclasses.make_dataclass("U8Row", [("v", object, blosc2.field(spec))]) + t = blosc2.CTable(Row, expected_size=8) + t.extend([([7, 200],), (None,)]) + assert t["v"].dtype == np.dtype(np.uint8) + assert t["v"][:][0].tolist() == [7, 200] + + +# --------------------------------------------------------------------------- +# Persistence, in place, and the crash ordering +# --------------------------------------------------------------------------- + + +def test_inplace_conversion_survives_a_reopen(tmp_path): + t = one_col([5, -1, 1], blosc2.int64(nullable=True, null_value=-1), urlpath=tmp_path / "t.b2t") + assert t.convert_nulls("a", to="mask", inplace=True) is t + assert t["a"].null_storage == "mask" + + reopened = blosc2.open(str(tmp_path / "t.b2t")) + assert reopened["a"].null_storage == "mask" + assert reopened._null_mask("a") is not None + assert as_list(reopened["a"]) == [5, None, 1] + + +def test_inplace_conversion_back_to_sentinel_survives_a_reopen(tmp_path): + t = one_col([5, None, 1], blosc2.int64(null_storage="mask"), urlpath=tmp_path / "t.b2t") + t.convert_nulls("a", to="sentinel", inplace=True) + reopened = blosc2.open(str(tmp_path / "t.b2t")) + assert reopened["a"].null_storage == "sentinel" + assert reopened._null_mask("a") is None + assert as_list(reopened["a"]) == [5, None, 1] + + +def test_inplace_on_a_persistent_table_refuses_a_dtype_change(tmp_path): + """No ordering of "replace the array" and "update the schema" is crash-safe.""" + t = one_col([1, 255, 0], blosc2.bool(nullable=True), urlpath=tmp_path / "t.b2t") + with pytest.raises(ValueError, match="changes its physical dtype"): + t.convert_nulls("a", to="mask", inplace=True) + assert t["a"].null_storage == "sentinel" + assert t["a"].dtype == np.dtype(np.uint8) + + +def test_the_same_column_converts_fine_out_of_place(tmp_path): + t = one_col([1, 255, 0], blosc2.bool(nullable=True), urlpath=tmp_path / "t.b2t") + converted = t.convert_nulls("a", to="mask") + assert converted["a"].dtype == np.dtype(np.bool_) + assert as_list(converted["a"]) == [True, None, False] + landed = converted.copy(urlpath=str(tmp_path / "out.b2d")) + assert landed["a"].null_storage == "mask" + assert as_list(landed["a"]) == [True, None, False] + + +def test_inplace_on_an_in_memory_table_may_change_dtype(): + """There is no crash window in memory, so the restriction does not apply.""" + t = one_col([1, 255, 0], blosc2.bool(nullable=True)) + t.convert_nulls("a", to="mask", inplace=True) + assert t["a"].dtype == np.dtype(np.bool_) + assert as_list(t["a"]) == [True, None, False] + + +def test_inplace_refuses_a_view(): + t = one_col([5, None, 1], blosc2.int64(null_storage="mask")) + with pytest.raises(ValueError, match="view"): + t.where("a > 0").convert_nulls("a", to="sentinel", inplace=True) + + +def test_inplace_refuses_a_read_only_table(tmp_path): + one_col([5, -1, 1], blosc2.int64(nullable=True, null_value=-1), urlpath=tmp_path / "t.b2t") + ro = blosc2.open(str(tmp_path / "t.b2t"), mode="r") + with pytest.raises(ValueError, match="read-only"): + ro.convert_nulls("a", to="mask", inplace=True) + + +def test_an_orphan_sidecar_still_reads_as_sentinel(tmp_path): + """The crash-safety argument for ``to="mask"``, in the state it protects. + + Step 1 writes the complete sidecar; step 2 flips the schema. A crash in + between leaves a ``.notnull`` key beside a schema that still says + ``sentinel`` -- and a sentinel column never opens a sidecar, so the table + reads exactly as it did before the conversion started. + """ + t = one_col([5, -1, 1], blosc2.int64(nullable=True, null_value=-1), urlpath=tmp_path / "t.b2t") + # Simulate an interruption after step 1 by writing the sidecar by hand. + mask = t._ensure_null_mask("a") + mask[:] = True + mask[1] = False + del t + + reopened = blosc2.open(str(tmp_path / "t.b2t"), mode="a") + assert reopened["a"].null_storage == "sentinel" + assert reopened._storage.has_null_mask("a") # the orphan is on disk + assert as_list(reopened["a"]) == [5, None, 1] # and is not consulted + # Finishing the conversion is then just the schema update. + reopened.convert_nulls("a", to="mask", inplace=True) + assert as_list(reopened["a"]) == [5, None, 1] + + +def test_a_mask_column_whose_nulls_still_hold_the_sentinel_reads_correctly(tmp_path): + """The other side of the same window: schema flipped, values not normalized. + + Step 3 rewrites the null slots to the fill and is purely cosmetic -- what + sits under ``valid=False`` is unobservable through the Column API, and the + fill is explicitly not part of the format contract. + """ + t = one_col([5, None, 1], blosc2.int64(null_storage="mask"), urlpath=tmp_path / "t.b2t") + t._cols["a"][1] = -12345 # as if step 3 had never run + assert t["a"].is_null().tolist() == [False, True, False] + assert t["a"].null_count() == 1 + assert t["a"].fillna(0).tolist() == [5, 0, 1] + assert t["a"].min() == 1 + + +# --------------------------------------------------------------------------- +# Nothing auto-migrates +# --------------------------------------------------------------------------- + + +def test_copy_preserves_each_columns_null_storage(): + t = table( + [(5, "ab"), (None, "__BLOSC2_NULL__")], + k=blosc2.int64(null_storage="mask"), + s=blosc2.string(max_length=2, nullable=True), + ) + c = t.copy() + assert c["k"].null_storage == "mask" + assert c["s"].null_storage == "sentinel" + + +def test_saving_preserves_null_storage_under_a_mask_default_policy(tmp_path): + """A policy governs *creation*; it must never rewrite what already exists.""" + t = one_col([5, -1, 1], blosc2.int64(nullable=True, null_value=-1)) + with blosc2.null_policy(blosc2.NullPolicy(null_storage="mask")): + t.save(str(tmp_path / "t.b2d")) + reopened = blosc2.open(str(tmp_path / "t.b2d")) + assert reopened["a"].null_storage == "sentinel" + assert reopened["a"].null_value == -1 + + +def test_opening_an_old_table_changes_nothing(tmp_path): + t = one_col([1, 255, 0], blosc2.bool(nullable=True), urlpath=tmp_path / "t.b2t") + del t + with blosc2.null_policy(blosc2.NullPolicy(null_storage="mask")): + reopened = blosc2.open(str(tmp_path / "t.b2t")) + assert reopened["a"].null_storage == "sentinel" + assert reopened["a"].dtype == np.dtype(np.uint8) + assert reopened["a"].null_value == 255 + + +# --------------------------------------------------------------------------- +# Detection +# --------------------------------------------------------------------------- + + +def test_null_storage_is_reported_per_column(): + t = table( + [(5, "ab", 1, "x")], + k=blosc2.int64(null_storage="mask"), + s=blosc2.string(max_length=2, nullable=True), + plain=blosc2.int64(), + d=blosc2.dictionary(), + ) + assert t["k"].null_storage == "mask" + assert t["s"].null_storage == "sentinel" + assert t["plain"].null_storage == "none" + assert t["d"].null_storage == "code" + + +def test_info_tags_each_column_with_where_its_nulls_live(): + """What you read to decide whether a table needs converting.""" + t = table( + [(5, "ab", 1)], + k=blosc2.int64(null_storage="mask"), + s=blosc2.string(max_length=2, nullable=True), + plain=blosc2.int64(), + ) + summary = dict(t.info_items)["columns"] + assert "nullable[mask]" in str(summary["k"]) + assert "nullable[sentinel]" in str(summary["s"]) + assert "nullable" not in str(summary["plain"]) + + +def test_column_info_reports_whether_a_sidecar_exists_yet(): + """The visible difference between "nullable" and "has ever held a null".""" + t = one_col([1, 2], blosc2.int64(null_storage="mask")) + assert dict(t["a"].info_items)["null_sidecar"] is False + t["a"][0] = None + assert dict(t["a"].info_items)["null_sidecar"] is True + + +# --------------------------------------------------------------------------- +# The converted table behaves like a natively created one +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("to", ["mask", "sentinel"]) +def test_a_converted_column_still_sorts_groups_and_queries(to): + """A converted column is indistinguishable from a natively created one.""" + if to == "mask": + t = one_col([5, -1, 1, 9, -1, 2], blosc2.int64(nullable=True, null_value=-1)) + else: + t = one_col([5, None, 1, 9, None, 2], blosc2.int64(null_storage="mask")) + converted = t.convert_nulls("a", to=to) + + assert as_list(converted.sort_by("a")["a"]) == [1, 2, 5, 9, None, None] + assert converted["a"].null_count() == 2 + assert converted["a"].min() == 1 + assert len(converted.where("a < 3")) == 2 + grouped = converted.group_by(["a"], dropna=True, sort=True).agg(n=("a", "count")) + assert len(grouped) == 4 + + +def test_a_converted_column_round_trips_through_arrow(): + """The point of converting to mask in the first place.""" + pa = pytest.importorskip("pyarrow") + t = one_col([True, 255, False], blosc2.bool(nullable=True)) + converted = t.convert_nulls("a", to="mask") + arrow = converted.to_arrow() + assert arrow.column("a").type == pa.bool_() + assert arrow.column("a").to_pylist() == [True, None, False] + + +def test_extending_a_converted_column_accepts_none(): + t = one_col([5, -1], blosc2.int64(nullable=True, null_value=-1)) + converted = t.convert_nulls("a", to="mask") + converted.append((None,)) + converted.extend([(7,), (None,)]) + assert as_list(converted["a"]) == [5, None, None, 7, None] From ec1bd76acfcfba1b473117c0adf930ea9ca148be Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 16:49:54 +0200 Subject: [PATCH 10/47] Make a validity sidecar the default (mask-based-nulls 9) A bare nullable=True now resolves to mask storage, as does every nullable column inferred from Arrow, Parquet or CSV. Lossless round-trip is the reason the sidecar exists, so it is what a newly created column should get: to_arrow o from_arrow is now the identity for nullable bool, full-range int8/uint8, floats holding nan/-0.0 as values, free-text utf8, and timestamps holding int64.min -- none of which a sentinel can represent, because a sentinel has to steal one. Nothing on disk changes. The default governs creation only; opening never re-resolves, so stored tables keep their storage, dtype and sentinel, and the rewrites for the reserved 255 stay permanently. A table with a mask column records schema version 3, which older readers refuse with a clear error. Not the one-line change the plan budgeted. NullPolicy.null_storage becomes tri-state -- None means unspecified -- because "the user asked for a mask" and "the user said nothing" have to stay distinguishable: setting a type-wide sentinel field still implies sentinel storage for the kinds it covers, and collapsing the two would have made NullPolicy(float_value=...) raise on the very release that flipped the default. bool_value is the one field that cannot imply anything, since 255 is both its default and the only value it may hold. Three gaps the flip exposed, all pre-existing and all fixed here: - CSV was sentinel-only. from_csv had nothing to put in an empty field and raised; to_csv compared against the sentinel and wrote a mask column's fill as if it were data. Both go through the sidecar now, in either direction. - `~` on a mask bool column selected its nulls: it inverted the stored False fill, where SQL says a null satisfies neither a predicate nor its negation. - The Arrow importer never consulted the type-wide sentinel inference, so NullPolicy(signed_int_strategy="max") meant one thing for a declared schema and another for an inferred one. The ~65 tests that wrote a sentinel literally to mean "null" now say null_storage="sentinel" and mean it; several gained a mask-side counterpart. Co-Authored-By: Claude Opus 5 --- RELEASE_NOTES.md | 54 +++++-- doc/reference/ctable.rst | 41 +++-- plans/mask-based-nulls.md | 23 ++- src/blosc2/ctable.py | 166 ++++++++++++++++---- src/blosc2/ctable_indexing.py | 5 +- tests/ctable/test_ctable_ndarray_columns.py | 12 +- tests/ctable/test_groupby.py | 6 +- tests/ctable/test_nested_metadata_root.py | 20 ++- tests/ctable/test_null_channel.py | 21 ++- tests/ctable/test_null_expressions.py | 2 +- tests/ctable/test_null_mask_arrow.py | 20 ++- tests/ctable/test_null_mask_sort_groupby.py | 2 +- tests/ctable/test_null_migration.py | 16 +- tests/ctable/test_null_storage_schema.py | 103 ++++++++++-- tests/ctable/test_nullable.py | 19 ++- tests/ctable/test_parquet_interop.py | 42 ++++- tests/ctable/test_utf8.py | 34 +++- 17 files changed, 468 insertions(+), 118 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 58e6831ea..d48ddf464 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,17 +6,18 @@ XXX version-specific blurb XXX ### New features -#### Mask-based nullable columns for CTable +#### Mask-based nullable columns for CTable, and they are now the default -A nullable CTable column can now keep its nulls in a per-column **validity -sidecar** — Arrow's own model — instead of reserving a value from its own range. -Pass `null_storage="mask"` on any scalar spec: +A nullable CTable column keeps its nulls in a per-column **validity sidecar** — +Arrow's own model — instead of reserving a value from its own range. This is now +what a bare `nullable=True` resolves to, and what every nullable column inferred +from Arrow, Parquet or CSV gets: ```python -blosc2.bool(null_storage="mask") # no reserved 255; dtype stays np.bool_ -blosc2.int8(null_storage="mask") # all 256 values usable, plus nulls -blosc2.utf8(null_storage="mask") # any string, including "" and "\x00" -blosc2.complex128(null_storage="mask") # nullable at all, for the first time +blosc2.bool(nullable=True) # no reserved 255; dtype stays np.bool_ +blosc2.int8(nullable=True) # all 256 values usable, plus nulls +blosc2.utf8(nullable=True) # any string, including "" and "\x00" +blosc2.complex128(nullable=True) # nullable at all, for the first time ``` This is what makes nullability **lossless**: a sentinel steals a value from the @@ -31,13 +32,27 @@ Under mask storage `None` is how you write a null (`t.append((None,))`, `t["price"][3] = None`), which a fixed-width sentinel column cannot accept at all. `is_null()` is unchanged and remains the uniform API across every kind. -Sentinel storage stays the **default** and is supported indefinitely; existing -tables open and behave exactly as before. `Column.null_storage` reports where a -column keeps its nulls and `info` tags each column (`int64 nullable[mask]`), so -`CTable.convert_nulls()` can move columns between the two in either direction — -never implicitly, and refusing rather than silently relabelling data when a -sentinel is unavailable. A table with a mask column records schema version 3, so -only such tables need a current reader. +**Nothing on disk changes.** The new default governs *creation* only: opening a +stored table never re-resolves anything, so every existing table keeps the +storage, dtype and sentinel it was written with, and every rewrite rule for the +reserved `255` stays permanently in place. Sentinel storage is supported +indefinitely and is one keyword away, per column (`null_storage="sentinel"`, or +any explicit `null_value=`) or globally through `NullPolicy`. Setting a type-wide +`NullPolicy` sentinel field still implies sentinel storage for the kinds it +covers, so existing `NullPolicy(float_value=...)` code is unaffected — with one +unavoidable exception: `255` is the only value a nullable bool may reserve, so it +is also `bool_value`'s default, and `NullPolicy(bool_value=255)` carries no +information to act on. A bool column that wants a sentinel has to say so with +`null_storage` or `column_null_values`. + +A table containing a mask column records **schema version 3**; readers older than +4.10.2 refuse it with a clear error rather than misreading it. Pass +`null_storage="sentinel"` for data that has to stay readable by them. + +`Column.null_storage` reports where a column keeps its nulls and `info` tags each +column (`int64 nullable[mask]`), so `CTable.convert_nulls()` can move columns +between the two in either direction — never implicitly, and refusing rather than +silently relabelling data when a sentinel is unavailable. One deliberate semantic difference: in a mask column `NaN` is a **value**, following Arrow, and only the sidecar marks a null. Sentinel float columns keep @@ -64,6 +79,15 @@ NaN-as-null. See "Where nulls are stored" in the CTable reference. widening that sentinel storage needs was undone by dtype rather than by whether it had been applied, so a column declared `uint8` was truncated to flags. +- **`~` on a nullable bool column selected its nulls.** SQL `WHERE` semantics + say a null satisfies neither a predicate nor its negation; the mask path + inverted the stored `False` fill instead. (The sentinel path was already + correct, via its `== 0` rewrite.) +- **CSV import and export ignored a validity sidecar.** `to_csv` compared + against the sentinel to find nulls, so a mask column wrote its fill as if it + were data, and `from_csv` had nothing to put in an empty field and raised. + Both go through the sidecar now: an empty CSV field is a null in either + direction. Sentinel columns keep writing their sentinel, unchanged. ## Changes from 4.10.0 to 4.10.1 diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index 6ae8c87d5..cfb559ebf 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -143,16 +143,27 @@ The two other values are the only representation their kind has: ``"code"`` for a dictionary column, which reserves ``-1``; ``"native"`` for the variable-length container kinds, whose cells simply hold ``None``. -Sentinel storage is the default and is supported indefinitely. Mask storage is -what makes nullability **lossless**, so it is worth asking for whenever data -comes from — or is going to — Arrow or Parquet: +**Mask storage is the default** since 4.10.2, because it is what makes +nullability lossless. A bare ``nullable=True`` — and every nullable column +inferred from Arrow, Parquet or CSV — keeps its nulls in a sidecar, so: .. code-block:: python - blosc2.bool(null_storage="mask") # no reserved 255; dtype stays np.bool_ - blosc2.int8(null_storage="mask") # all 256 values usable, plus nulls - blosc2.utf8(null_storage="mask") # any string, including "" and "\x00" - blosc2.complex128(null_storage="mask") # nullable at all, for the first time + blosc2.bool(nullable=True) # no reserved 255; dtype stays np.bool_ + blosc2.int8(nullable=True) # all 256 values usable, plus nulls + blosc2.utf8(nullable=True) # any string, including "" and "\x00" + blosc2.complex128(nullable=True) # nullable at all, for the first time + +Sentinel storage is supported indefinitely and is one keyword away, per column +(``null_storage="sentinel"``, or any explicit ``null_value=``) or globally +through :class:`NullPolicy`. It is the right choice when a column has to stay +readable by a Blosc2 release older than 4.10.2: a table containing a mask column +records **schema version 3**, which earlier readers refuse with a clear error +rather than misreading. + +Nothing on disk changes. The flip governs *creation* only — opening a stored +table never re-resolves anything, so every existing table keeps the storage, +dtype and sentinel it was written with. Under mask storage ``None`` is the way to write a null — ``t.append((None,))``, ``t["price"][3] = None`` — which a fixed-width sentinel column cannot accept at @@ -192,8 +203,8 @@ Null policy ----------- :class:`NullPolicy` decides what a bare ``nullable=True`` resolves to when the -schema does not say: which sentinel to reserve, or whether to use a mask at -all. Scope it with :func:`null_policy`:: +schema does not say: whether to use a mask at all, and if not, which sentinel to +reserve. Scope it with :func:`null_policy`:: policy = blosc2.NullPolicy( signed_int_strategy="max", @@ -226,9 +237,15 @@ sentinel itself comes from ``column_null_values`` if listed and from the type-wide default otherwise. Setting any type-wide sentinel field therefore *implies* sentinel storage for -the kinds it covers, so ``NullPolicy(float_value=-1.0)`` keeps working as it -always has. Passing ``null_storage="mask"`` alongside one is the one -combination that raises, because it contradicts itself. +the kinds it covers, so ``NullPolicy(float_value=-1.0)`` keeps working exactly as +it did before the default flipped. Passing ``null_storage="mask"`` alongside one +is the one combination that raises, because it contradicts itself. + +``bool_value`` is the exception, and unavoidably so: ``255`` is the only value a +nullable bool may reserve, so it is also the field's default, and +``NullPolicy(bool_value=255)`` carries no information to act on. A bool column +that wants sentinel storage has to say so with ``null_storage`` or +``column_null_values``. Columns without ``nullable=True``, an explicit ``null_value`` or an explicit ``null_storage`` are not nullable. diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index acda5bf99..c2fcc33a5 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,9 +1,9 @@ # Mask-based nullable columns for CTable -> **Status: IN PROGRESS — Phases 0–8 landed 2026-08-08.** Lossless Arrow/Parquet round-trip works -> for every V1 kind, sort/groupby/query honour a sidecar, and `convert_nulls` migrates columns in -> either direction — all **opt-in** via `null_storage="mask"`; next up is Phase 9 (the default -> flip). Six premises were disproven during +> **Status: Phases 0–9 landed 2026-08-08; only Phase 10 remains.** Lossless Arrow/Parquet +> round-trip works for every V1 kind, sort/groupby/query honour a sidecar, `convert_nulls` migrates +> columns in either direction, and **mask storage is now the default** — a bare `nullable=True` +> resolves to it. Six premises were disproven during > implementation and are corrected in place, each in a blockquote beside the text it corrects: the > index path cannot be fixed by a null-aware expression (§Expression layer), the bool dtype-flip > cannot move out of `__init__` (§Schema layer), ndarray columns do not get lazy null propagation @@ -60,7 +60,8 @@ answers. 1. **Mask becomes the default** for `nullable=True` on newly created tables — in two steps: the capability ships opt-in first (Phase 6), and the default flips no earlier than one release later (Phase 9), so version-3-capable readers are in circulation before - default-created tables require them. Sentinel remains fully supported and readable forever, + default-created tables require them. *(The two-step staging was not kept — both shipped in + 4.10.2; see the deviation note under §Phasing.)* Sentinel remains fully supported and readable forever, selectable per column (`null_value=...`, `null_storage="sentinel"`) or globally via `NullPolicy`. Existing on-disk tables keep working unchanged. 2. **V1 scope** = fixed-width scalars + utf8: numeric (incl. **complex**, which gains nullability @@ -889,13 +890,23 @@ default-created tables require them. | 6 | ✅ **Arrow/Parquet.** Import + export for all V1 kinds; `arrow_slice(valid=)`; `null_storage=` on `from_arrow`/`from_parquet`; the "no sentinel available" import error now names the way out instead of being deleted (it still fires for sentinel storage, which still cannot represent those types). No `packbits` — pyarrow's own packing is borrowed instead. Ships **opt-in**; the default stays `"sentinel"`. `tests/ctable/test_null_mask_arrow.py` (42 tests). | M | Med | | 7 | ✅ **Sort + groupby.** `_build_lex_keys` (the indicator key is nulls-last *entirely*, not a refinement), `_sorted_positions_from_full_index` (3.0x for `U16`, 1.2x for `int64` — the I/O win is in bytes, not proportionally in time), `_utf8_rank_arrays(valid=)` plus a `null_aware` staleness rule no O(1) signal could replace, `_sorted_slice_positions` bails, groupby `_null_mask(valid=)` **plus `_CodedKeyChunk`**, which this section had not anticipated: a mask *key* column needs a reserved null code, not a threaded flag. **Plus the mask half of Phase 1**, which had never been done — `where()` leaked nulls on both the scan and the index (see §Expression layer, Addendum 2). Three pre-existing storage-independent bugs fixed on the way: descending sort of `bool` (raised) and of full-range signed ints (wrong order), and groupby `min`/`max` over `bool` (always `False`). `tests/ctable/test_null_mask_sort_groupby.py` (75 tests). | **L** (was M) | Med-High | | 8 | ✅ **Migration + docs.** `convert_nulls` both directions for every V1 kind, refusing what a sentinel cannot represent; the crash ordering **corrected** (fill after the schema flip, not before) and asserted; a persistent in-place dtype change refused with a reason; `_detach_schema` so a converted copy stops relabelling its source; the table-level `info` null tag (`Column.null_storage` and the per-column `info` rows already existed). `doc/reference/ctable.rst` gains a "Where nulls are stored" section and a rewritten null-policy resolution order; release notes. Three storage-independent bugs fixed on the way: `copy()`'s off-by-one write watermark (which *raised* in `add_column`), and a nullable `uint8` ndarray column coming back as `bool_`. `tests/ctable/test_null_migration.py` (50 tests). | M (was S–M) | Low | -| 9 | **Default flips to `"mask"`.** A one-line `NullPolicy` change plus release notes — lossless round-trip is why the default exists. Lands **no earlier than one release after Phase 6** so older readers in the wild already understand schema version 3. | S | Low | +| 9 | ✅ **Default flips to `"mask"`.** Not a one-line change: `null_storage` had to become tri-state (`None` = unspecified) so that a type-wide sentinel field can still imply sentinel storage without contradicting the new default, and ~65 tests that wrote a sentinel literally to mean "null" had to say `null_storage="sentinel"` and mean it. Three real gaps the flip exposed, all fixed: **CSV import/export was sentinel-only** (`from_csv` raised on an empty field, `to_csv` wrote the fill as data), **`~` on a mask bool column selected its nulls**, and **the Arrow importer ignored the type-wide sentinel inference**, so `NullPolicy(signed_int_strategy="max")` meant one thing for a declared schema and another for an inferred one. Landed in the **same** session as Phase 6, not a release later — see the note below. | M (was S) | Med (was Low) | | 10 | **Index null-awareness remainder** *(independent)*. Mask-aware summary builder; `null_aware`/`null_order` descriptors; re-enable `_summary_minmax_source` for mask and sentinel columns alike. | **L** | High | The riskiest, most-coupled work is isolated into Phases 4, 7 and 10, each of which can slip without blocking the others. Phase 9 is a policy change, not code — its only prerequisite is that Phases 2–8 have soaked for a release. +> **Deviation from decision 1 (2026-08-08), recorded deliberately.** Phase 9 landed in the same +> session as Phase 6, not a release later. The staging existed so that version-3-capable readers +> would be in circulation before default-created tables required them; shipping both at once means +> the first release carrying the flip is also the first release able to read what it writes. The +> mitigation is unchanged and was always the real safety net: the version bump is **conditional on a +> mask column existing**, so an older reader meets a clear `ValueError: Unsupported schema version 3` +> rather than misreading anything, and `null_storage="sentinel"` remains one keyword away for data +> that has to stay readable by them. Flagged to the maintainer at the time as a release-scheduling +> call rather than a technical one. + ## Named follow-ups (not blocking any phase) - **Validity through the Cython groupby kernels.** A mask column holding a null costs ~1.9x on a diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 5631d509c..b88bf313e 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -128,13 +128,18 @@ def _is_arrow_binary_type(pa, pa_type) -> bool: @dataclass(frozen=True) class NullPolicy: - """Default sentinels for inferred CTable scalar nulls. + """Where a nullable CTable column keeps its nulls, and which sentinel if in band. - CTable nullable scalar columns are represented with per-column sentinel - values. This policy is used when CTable has to infer those sentinels, such - as when importing nullable scalar Arrow or Parquet columns without an - explicit column-level null sentinel. The selected sentinel is stored in the - resulting CTable schema, so existing tables remain self-describing. + Consulted whenever a schema does not say -- a bare ``nullable=True``, or a + nullable Arrow/Parquet/CSV column being inferred. Whatever it decides is + written into the resulting schema, so a stored table stays self-describing + and is never re-resolved on open. + + Since 4.10.2 the default is a **validity sidecar** (``null_storage="mask"``), + which is what makes nullability lossless. Setting any type-wide sentinel + field below asks for in-band storage for the kinds that field covers, so + existing ``NullPolicy(float_value=...)`` code keeps its sentinels; passing + ``null_storage="mask"`` alongside one is a contradiction and raises. Examples -------- @@ -163,6 +168,12 @@ class Row: ``column_null_values`` takes precedence over the type-wide defaults in the policy. This is useful when a particular column needs a sentinel that is known not to collide with its real values. + + ``bool_value`` cannot imply sentinel storage the way the other fields do: + ``255`` is the only value a nullable bool may reserve, so it is also this + field's default, and ``NullPolicy(bool_value=255)`` states the default rather + than choosing anything. A bool column that wants a sentinel says so with + ``null_storage="sentinel"`` or ``column_null_values``. """ string_value: str = "__BLOSC2_NULL__" @@ -173,7 +184,18 @@ class Row: unsigned_int_strategy: Literal["min", "max"] = "max" timestamp_value: int = int(np.iinfo(np.int64).min) column_null_values: Mapping[str, Any] = dataclass_field(default_factory=dict) - null_storage: Literal["mask", "sentinel"] = "sentinel" + #: Where a bare ``nullable=True`` keeps its nulls. ``None`` means *not + #: specified*, which resolves to :attr:`DEFAULT_NULL_STORAGE` -- and which + #: has to stay distinct from an explicit ``"mask"``, so that setting a + #: type-wide sentinel field can imply sentinel storage for the kinds it + #: covers without contradicting anything the caller wrote. + null_storage: Literal["mask", "sentinel"] | None = None + + #: What an unspecified ``null_storage`` resolves to. ``"mask"`` since + #: 4.10.2: lossless nullability is why the sidecar exists, so it is what a + #: newly created nullable column should get. Sentinel storage stays fully + #: supported and is one kwarg away. + DEFAULT_NULL_STORAGE: ClassVar[str] = NULL_MASK #: Type-wide sentinel fields, paired with the spec attribute that says #: which columns each one covers. Setting any of them is what makes a @@ -188,16 +210,27 @@ class Row: "timestamp_value", ) + def resolve_null_storage(self) -> str: + """Where a bare ``nullable=True`` puts its nulls under this policy. + + An unspecified :attr:`null_storage` resolves to + :attr:`DEFAULT_NULL_STORAGE` here rather than at construction, so that + ``NullPolicy()`` keeps meaning "whatever this version's default is" + however the policy is copied around. + """ + return self.DEFAULT_NULL_STORAGE if self.null_storage is None else self.null_storage + def __post_init__(self): - if self.null_storage not in ("mask", "sentinel"): - raise ValueError(f"null_storage must be 'mask' or 'sentinel', got {self.null_storage!r}") + if self.null_storage is not None and self.null_storage not in ("mask", "sentinel"): + raise ValueError(f"null_storage must be 'mask', 'sentinel' or None, got {self.null_storage!r}") # Setting a type-wide sentinel field alongside an explicit # null_storage="mask" is a contradiction the caller wrote down, so say # so. Setting one *without* an explicit null_storage is not: it simply # means "use sentinels for the types I named", which is what # _resolve_nullable_specs does with it. Raising on that instead would # break existing NullPolicy(float_value=...) code on the very release - # that flips the default. + # that flipped the default -- which is exactly why an unspecified + # null_storage stays None rather than being resolved here. if self.null_storage == "mask": explicit = [f for f in self._SENTINEL_FIELDS if self._sentinel_field_is_set(f)] if explicit: @@ -2141,7 +2174,18 @@ def __invert__(self): self._ensure_queryable() if self._is_nullable_bool: return self._raw_col == 0 - return ~self._raw_col + inverted = ~self._raw_col + if self.dtype == np.dtype(np.bool_): + valid = self._nulls.valid_pred() + if valid is not None: + # A mask-backed nullable bool holds the ``False`` fill in its + # null rows, and ``~False`` is True -- so without this, negating + # a nullable flag *selects* its nulls. SQL WHERE semantics: a + # null satisfies neither the predicate nor its negation. The + # sentinel branch above gets this from ``== 0``, which excludes + # the reserved 255 as well as the true rows. + return inverted & valid + return inverted def __lt__(self, other): if self.is_utf8: @@ -4784,7 +4828,7 @@ def _resolved_null_storage(cls, name: str, spec, policy) -> str: return NULL_MASK if cls._policy_implies_sentinel(spec, policy): return NULL_SENTINEL - return policy.null_storage + return policy.resolve_null_storage() @staticmethod def _unflip_mask_bool_dtype(spec) -> None: @@ -7935,7 +7979,7 @@ def _compiled_columns_from_arrow( null_policy = get_null_policy() # Only inferred schemas consult the policy; extending an existing table # never reaches here, so its stored null_storage always wins. - storage_pref = null_storage if null_storage is not None else null_policy.null_storage + storage_pref = null_storage if null_storage is not None else null_policy.resolve_null_storage() column_null_values = null_policy.column_null_values schema_names = set(schema.names) unknown_null_values = set(column_null_values) - schema_names @@ -8007,6 +8051,24 @@ def _compiled_columns_from_arrow( and not handles_own_nulls and not has_null_value_override ) + if use_mask and null_storage is None: + # A type-wide policy sentinel field is a request for in-band + # storage for the kinds it covers, and has to be honoured here + # too -- otherwise NullPolicy(signed_int_strategy="max") would + # mean one thing for a declared schema and another for an + # inferred one. Asking needs a spec, so build a throwaway with + # no null decision in it; an explicit null_storage= argument + # has already won by this point. + probe = cls._arrow_type_to_spec( + pa, + field.type, + arrow_col, + field_metadata=field.metadata, + string_max_length=column_string_max_length, + nullable=field.nullable, + object_fallback=object_fallback, + ) + use_mask = not cls._policy_implies_sentinel(probe, null_policy) if has_null_value_override: null_value = column_null_values[name] elif not use_mask and auto_null_sentinels and field.nullable and not handles_own_nulls: @@ -9223,7 +9285,9 @@ def to_csv(self, path: str | None = None, *, header: bool = True, sep: str = "," col = self[name] if col.is_ndarray: arr = col[:] - null_mask = col._null_mask_for(arr) + # is_null(), not a sentinel comparison: a mask column's nulls are + # not in its values at all, and its fill is an ordinary item. + null_mask = col.is_null() json_strings: list[str] = [] for i in range(n): if null_mask[i]: @@ -9231,6 +9295,14 @@ def to_csv(self, path: str | None = None, *, header: bool = True, sep: str = "," else: json_strings.append(json.dumps(arr[i].tolist())) arrays.append(json_strings) + elif col.null_storage == NULL_MASK and col.null_count(): + # An empty field is CSV for missing, and for a mask column it is + # the *only* way to say it -- writing the fill would come back as + # a real 0 or "". A sentinel column keeps writing its sentinel, + # which from_csv maps back, so its output is unchanged. + arrays.append( + ["" if is_null else v for v, is_null in zip(col[:], col.is_null(), strict=True)] + ) else: arrays.append(col[:]) @@ -9250,20 +9322,33 @@ def _write(f) -> None: return None @staticmethod - def _csv_ndarray_col_to_array(raw: list[str], col) -> np.ndarray: - """Convert a list of JSON-array CSV strings to a stacked ndarray for an ndarray column.""" + def _csv_ndarray_col_to_array(raw: list[str], col) -> tuple[np.ndarray, np.ndarray | None]: + """Convert JSON-array CSV strings to ``(values, valid)`` for an ndarray column. + + Same split as :meth:`_csv_col_to_array`: an empty field is missing, and a + mask column reports that in *valid* rather than in the item it stores. + """ spec = col.spec null_value = getattr(spec, "null_value", None) + uses_mask = getattr(spec, "uses_mask", False) item_shape = spec.item_shape dtype = spec.dtype + fill = None if not uses_mask else np.full(item_shape, fill_value_for(spec), dtype=dtype) + valid = None rows = [] - for val in raw: + for i, val in enumerate(raw): stripped = val.strip() if stripped == "": if null_value is not None: rows.append(np.full(item_shape, null_value, dtype=dtype)) continue + if fill is not None: + if valid is None: + valid = np.ones(len(raw), dtype=np.bool_) + valid[i] = False + rows.append(fill) + continue raise ValueError(f"Column {col.name!r}: non-nullable column got empty cell") try: @@ -9275,25 +9360,42 @@ def _csv_ndarray_col_to_array(raw: list[str], col) -> np.ndarray: raise ValueError(f"Column {col.name!r}: expected item shape {item_shape}, got {arr.shape}") rows.append(arr) - return np.ascontiguousarray(rows, dtype=dtype) + return np.ascontiguousarray(rows, dtype=dtype), valid @staticmethod - def _csv_col_to_array(raw: list[str], col, nv) -> np.ndarray: - """Convert a list of raw CSV strings to a numpy array for *col*.""" + def _csv_col_to_array(raw: list[str], col, nv) -> tuple[np.ndarray, np.ndarray | None]: + """Convert raw CSV strings to ``(values, valid)`` for *col*. + + An empty field is CSV for "missing". A sentinel column stores *nv* + there and is done; a **mask** column has no in-band value to store, so + it takes the column's fill and reports the validity separately -- and + without that a bare ``""`` would be cast straight to the column's dtype + and raise. *valid* is ``None`` when nothing was missing, or when the + column is not nullable at all. + """ + missing = np.array([v.strip() == "" for v in raw], dtype=np.bool_) + placeholder = nv + valid = None + if nv is None and getattr(col.spec, "uses_mask", False) and missing.any(): + placeholder = fill_value_for(col.spec) + valid = ~missing + if col.dtype == np.bool_: - def _parse(v, _nv=nv): + def _parse(v, _fill=placeholder): stripped = v.strip() - if stripped == "" and _nv is not None: - return _nv + if stripped == "" and _fill is not None: + return _fill return stripped in ("True", "true", "1") - return np.array([_parse(v) for v in raw], dtype=np.bool_) + return np.array([_parse(v) for v in raw], dtype=np.bool_), valid if col.dtype.kind == "S": - prepared: list = [nv if (v.strip() == "" and nv is not None) else v.encode() for v in raw] - return np.array(prepared, dtype=col.dtype) - prepared2 = [nv if (v.strip() == "" and nv is not None) else v for v in raw] - return np.array(prepared2, dtype=col.dtype) + prepared: list = [ + placeholder if (v.strip() == "" and placeholder is not None) else v.encode() for v in raw + ] + return np.array(prepared, dtype=col.dtype), valid + prepared2 = [placeholder if (v.strip() == "" and placeholder is not None) else v for v in raw] + return np.array(prepared2, dtype=col.dtype), valid @classmethod def from_csv( @@ -9410,11 +9512,15 @@ def from_csv( if n > 0: for i, col in enumerate(schema.columns): if isinstance(col.spec, NDArraySpec): - arr = cls._csv_ndarray_col_to_array(col_data[i], col) + arr, valid = cls._csv_ndarray_col_to_array(col_data[i], col) else: nv = getattr(col.spec, "null_value", None) - arr = cls._csv_col_to_array(col_data[i], col, nv) + arr, valid = cls._csv_col_to_array(col_data[i], col, nv) new_cols[col.name][:n] = arr + if valid is not None: + # A mask column's empty cells: the values above hold the + # fill, and this is the only record that they were missing. + obj._ensure_null_mask(col.name)[:n] = valid new_valid[:n] = True obj._n_rows = n obj._last_pos = n diff --git a/src/blosc2/ctable_indexing.py b/src/blosc2/ctable_indexing.py index 84291ed67..e4b359ccf 100644 --- a/src/blosc2/ctable_indexing.py +++ b/src/blosc2/ctable_indexing.py @@ -173,7 +173,10 @@ def _persist_utf8_vocab(full: dict, meta: dict, sorted_vocab: np.ndarray) -> Non if values_path is None: # in-memory index meta["vocab"] = sorted_vocab.tolist() return - width = max(len(v) for v in sorted_vocab) + # max(1, ...): a mask-storage column's ``""`` fill is an ordinary vocabulary + # entry, so an all-null column factorizes to exactly ``[""]`` -- and ``", blosc2.string(max_length=4, nullable=True), str), ("bytes_value", b"", blosc2.bytes(max_length=4, nullable=True), bytes), ("float_value", -1.0, blosc2.float64(nullable=True), float), - ("bool_value", 255, blosc2.bool(nullable=True), bool), ("timestamp_value", -1, blosc2.timestamp(nullable=True), object), ("signed_int_strategy", "max", blosc2.int64(nullable=True), int), ("unsigned_int_strategy", "min", blosc2.uint64(nullable=True), int), @@ -276,6 +283,20 @@ def test_type_wide_sentinel_field_implies_sentinel_for_its_kinds(field, value, s assert col.spec.null_value is not None +def test_bool_value_cannot_imply_sentinel_storage(): + """``bool_value``'s default is the only value it may hold, so "was it set?" + has no answer for it: ``255`` is the sole legal sentinel for a nullable + bool. ``NullPolicy(bool_value=255)`` therefore states the default and + changes nothing -- a bool column that wants sentinel storage has to say so, + with ``null_storage`` or ``column_null_values``. + """ + col = _resolved(blosc2.bool(nullable=True), bool, bool_value=255) + assert col.spec.uses_mask + col = _resolved(blosc2.bool(nullable=True), bool, null_storage="sentinel") + assert not col.spec.uses_mask + assert col.spec.null_value == 255 + + def test_policy_mask_applies_to_plain_nullable(): col = _resolved(blosc2.int64(nullable=True), null_storage="mask") assert col.spec.uses_mask @@ -287,7 +308,7 @@ def test_mask_skips_string_max_length_widening(): col = _resolved(blosc2.string(max_length=4, nullable=True), str, null_storage="mask") assert col.dtype == np.dtype("U4") - sentinel_col = _resolved(blosc2.string(max_length=4, nullable=True), str) + sentinel_col = _resolved(blosc2.string(max_length=4, nullable=True), str, null_storage="sentinel") assert sentinel_col.dtype == np.dtype("U15") @@ -303,9 +324,13 @@ def test_mask_skips_the_ndarray_bool_uint8_flip(): assert col.spec.itemsize == 1 -def test_sentinel_default_is_unchanged(): - """The whole point of shipping opt-in first: nothing moves by default.""" - col = _resolved(blosc2.bool(nullable=True), bool) +def test_sentinel_storage_is_unchanged_when_asked_for(): + """The uint8 widening and the reserved 255 stay exactly as they were. + + Every stored table keeps them, and so does any column that asks -- which is + what makes the default flip a change to *new* columns only. + """ + col = _resolved(blosc2.bool(nullable=True), bool, null_storage="sentinel") assert col.dtype == np.dtype(np.uint8) assert col.spec.null_value == 255 @@ -402,3 +427,63 @@ def test_storage_survives_to_cframe(): def test_storage_survives_copy(): """Nothing auto-migrates: copy() preserves each column's storage.""" assert _storage_of(_mask_table().copy()) == _EXPECTED_STORAGE + + +# --------------------------------------------------------------------------- +# The default flip (4.10.2) +# --------------------------------------------------------------------------- + + +def test_a_bare_nullable_column_gets_a_mask(): + """What the flip changes, stated once for each kind that has a choice.""" + for spec, annotation in [ + (blosc2.int8(nullable=True), int), + (blosc2.float64(nullable=True), float), + (blosc2.bool(nullable=True), bool), + (blosc2.string(max_length=4, nullable=True), str), + (blosc2.bytes(max_length=4, nullable=True), bytes), + (blosc2.utf8(nullable=True), str), + (blosc2.timestamp(nullable=True), object), + ]: + col = _resolved(spec, annotation) + assert col.spec.uses_mask, spec + assert col.spec.null_value is None, spec + + +def test_a_stored_table_keeps_the_storage_it_was_written_with(tmp_path): + """The flip governs *creation* only; opening never re-resolves anything. + + Which is the whole reason it is safe: every table already on disk carries + its own answer, sentinel included, and reading one does not consult a policy. + """ + Row = dataclasses.make_dataclass( + "OldRow", + [("flag", bool, blosc2.field(blosc2.bool(nullable=True, null_storage="sentinel")))], + ) + t = blosc2.CTable(Row, urlpath=str(tmp_path / "t.b2t"), mode="w", expected_size=4) + t.extend([(1,), (255,), (0,)]) + del t + + reopened = blosc2.open(str(tmp_path / "t.b2t")) + assert reopened["flag"].null_storage == "sentinel" + assert reopened["flag"].dtype == np.dtype(np.uint8) + assert reopened["flag"].null_value == 255 + assert reopened["flag"].is_null().tolist() == [False, True, False] + + +def test_a_mask_default_table_records_schema_version_3(tmp_path): + """Only a table that *uses* a mask needs a reader that understands one.""" + from blosc2.schema_compiler import schema_to_dict + + Row = dataclasses.make_dataclass("V3Row", [("v", int, blosc2.field(blosc2.int64(nullable=True)))]) + t = blosc2.CTable(Row, expected_size=4) + assert schema_to_dict(t._schema)["version"] == 3 + + Plain = dataclasses.make_dataclass("V1Row", [("v", int, blosc2.field(blosc2.int64()))]) + assert schema_to_dict(blosc2.CTable(Plain, expected_size=4)._schema)["version"] == 1 + + +def test_the_flip_is_one_kwarg_from_the_old_behaviour(): + col = _resolved(blosc2.int64(nullable=True), int, null_storage="sentinel") + assert not col.spec.uses_mask + assert col.spec.null_value == np.iinfo(np.int64).min diff --git a/tests/ctable/test_nullable.py b/tests/ctable/test_nullable.py index 779fe34c3..1a7db3880 100644 --- a/tests/ctable/test_nullable.py +++ b/tests/ctable/test_nullable.py @@ -107,14 +107,21 @@ def test_null_value_string(): def test_nullable_true_uses_default_null_policy(): + """The type-wide sentinels a policy picks, once sentinel storage is asked for. + + A bare ``nullable=True`` resolves to a mask since 4.10.2, so this pins the + *sentinel* half of the resolution -- which is still what + ``null_storage="sentinel"`` and any type-wide policy field select. + """ + @dataclass class Row: - i: int = blosc2.field(blosc2.int32(nullable=True)) - u: int = blosc2.field(blosc2.uint32(nullable=True)) - f: float = blosc2.field(blosc2.float64(nullable=True)) - flag: bool = blosc2.field(blosc2.bool(nullable=True)) - s: str = blosc2.field(blosc2.string(max_length=4, nullable=True)) - b: bytes = blosc2.field(blosc2.bytes(max_length=4, nullable=True)) + i: int = blosc2.field(blosc2.int32(nullable=True, null_storage="sentinel")) + u: int = blosc2.field(blosc2.uint32(nullable=True, null_storage="sentinel")) + f: float = blosc2.field(blosc2.float64(nullable=True, null_storage="sentinel")) + flag: bool = blosc2.field(blosc2.bool(nullable=True, null_storage="sentinel")) + s: str = blosc2.field(blosc2.string(max_length=4, nullable=True, null_storage="sentinel")) + b: bytes = blosc2.field(blosc2.bytes(max_length=4, nullable=True, null_storage="sentinel")) t = CTable(Row) assert t["i"].null_value == np.iinfo(np.int32).min diff --git a/tests/ctable/test_parquet_interop.py b/tests/ctable/test_parquet_interop.py index 908becedb..cff49f058 100644 --- a/tests/ctable/test_parquet_interop.py +++ b/tests/ctable/test_parquet_interop.py @@ -430,8 +430,7 @@ def test_utf8_arrow_roundtrip_no_singleton_list(self): out = t.to_arrow() if HAVE_STRING_DTYPE: assert t["txt"].is_utf8 - nv = t["txt"].null_value - assert list(t["txt"][:]) == ["short", long_str, nv, "end"] + assert t["txt"].is_null().tolist() == [False, False, True, False] # Export back to Arrow → still a scalar string column, not list assert pa.types.is_large_string(out.schema.field("txt").type) else: @@ -467,8 +466,7 @@ def test_utf8_parquet_roundtrip(self, tmp_path): assert t["txt"].is_varlen_scalar if HAVE_STRING_DTYPE: assert t["txt"].is_utf8 - nv = t["txt"].null_value - assert list(t["txt"][:]) == ["short", long_str, nv, "end"] + assert t["txt"].is_null().tolist() == [False, False, True, False] else: assert not t["txt"].is_utf8 # vlstring fallback, native-None nulls assert list(t["txt"][:]) == ["short", long_str, None, "end"] @@ -507,12 +505,25 @@ class NullableListRow: assert t2["vals"][2] == [3] def test_scalar_null_no_sentinel_raises(self, tmp_path): - """Importing Parquet scalar nulls without a null_value sentinel fails.""" + """Under *sentinel* storage, nulls with no sentinel available still fail. + + The default is mask storage, which needs no sentinel and so has no such + failure mode -- see :meth:`test_scalar_null_defaults_to_a_mask`. + """ at = pa.table({"score": pa.array([1.0, None, 3.0], type=pa.float64())}) path = tmp_path / "nulls.parquet" pq.write_table(at, path) with pytest.raises(TypeError, match="null_value sentinel"): - CTable.from_parquet(path, auto_null_sentinels=False) + CTable.from_parquet(path, auto_null_sentinels=False, null_storage="sentinel") + + def test_scalar_null_defaults_to_a_mask(self, tmp_path): + """No sentinel to choose, and none needed: nullity goes in the sidecar.""" + at = pa.table({"score": pa.array([1.0, None, 3.0], type=pa.float64())}) + path = tmp_path / "nulls.parquet" + pq.write_table(at, path) + t = CTable.from_parquet(path, auto_null_sentinels=False) + assert t["score"].null_storage == "mask" + assert t["score"].is_null().tolist() == [False, True, False] def test_scalar_null_exported_as_parquet_null(self, tmp_path): """Sentinel values become Parquet nulls on export.""" @@ -546,7 +557,10 @@ def test_auto_nullable_scalars_roundtrip(self, tmp_path): assert t["s"].null_count() == 1 assert t["b"].null_count() == 1 assert t["flag"].null_count() == 1 - assert t["flag"][:].tolist() == [1, 255, 0] + # A mask-backed nullable bool is a real np.bool_ column: no reserved 255, + # and the null slot holds the (unobservable) False fill. + assert t["flag"].dtype == np.dtype(np.bool_) + assert t["flag"].is_null().tolist() == [False, True, False] out = tmp_path / "nullable_scalars_out.parquet" t.to_parquet(out) rt = pq.read_table(out) @@ -653,7 +667,7 @@ def test_nullable_bool_filter_semantics(self, tmp_path): at = pa.table({"flag": pa.array([True, None, False], type=pa.bool_())}) path = tmp_path / "nullable_bool.parquet" pq.write_table(at, path) - t = CTable.from_parquet(path) + t = CTable.from_parquet(path, null_storage="sentinel") assert t.where(t.flag).flag[:].tolist() == [1] assert t.where(~t.flag).flag[:].tolist() == [0] assert t.where(t.flag == True).flag[:].tolist() == [1] # noqa: E712 @@ -661,6 +675,18 @@ def test_nullable_bool_filter_semantics(self, tmp_path): assert t.where(t.flag != True).flag[:].tolist() == [0] # noqa: E712 assert t.where(t.flag != False).flag[:].tolist() == [1] # noqa: E712 + def test_nullable_bool_filter_semantics_under_a_mask(self, tmp_path): + """The same semantics with no reserved 255 and no rewrite to get there.""" + at = pa.table({"flag": pa.array([True, None, False], type=pa.bool_())}) + path = tmp_path / "nullable_bool_mask.parquet" + pq.write_table(at, path) + t = CTable.from_parquet(path) + assert t["flag"].dtype == np.dtype(np.bool_) + assert t.where(t.flag).flag[:].tolist() == [True] + assert t.where(~t.flag).flag[:].tolist() == [False] + assert t.where(t.flag == True).flag[:].tolist() == [True] # noqa: E712 + assert t.where(t.flag == False).flag[:].tolist() == [False] # noqa: E712 + # --------------------------------------------------------------------------- # Error handling diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 13bdd3a07..e0d7bfa29 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -32,7 +32,7 @@ class Row: @dataclass class NullableRow: - name: str = blosc2.field(blosc2.utf8(nullable=True)) + name: str = blosc2.field(blosc2.utf8(nullable=True, null_storage="sentinel")) x: int = blosc2.field(blosc2.int64()) @@ -712,8 +712,8 @@ def test_ctable_utf8_comparison_excludes_null_rows(): def test_ctable_utf8_column_vs_column_comparison(): @dataclass class TwoCols: - a: str = blosc2.field(blosc2.utf8(nullable=True)) - b: str = blosc2.field(blosc2.utf8(nullable=True)) + a: str = blosc2.field(blosc2.utf8(nullable=True, null_storage="sentinel")) + b: str = blosc2.field(blosc2.utf8(nullable=True, null_storage="sentinel")) t = CTable(TwoCols, new_data={"a": ["x", "y", None, "z"], "b": ["x", "z", "q", None]}) eq = t[t.a == t.b] @@ -1296,15 +1296,31 @@ def test_utf8_from_arrow_large_string_ingest(): def test_utf8_from_arrow_nulls_use_sentinel(): + """Importing under an explicitly sentinel policy still reserves a string. + + The *default* is mask storage now, which is what makes a free-text utf8 + column round-trip at all (any string is a legal value, so no sentinel is + safe); this pins that asking for the old behaviour still gets it. + """ pa = pytest.importorskip("pyarrow") at = pa.table({"name": pa.array(["a", None, "c"], type=pa.string())}) - t = CTable.from_arrow(at.schema, at.to_batches()) + t = CTable.from_arrow(at.schema, at.to_batches(), null_storage="sentinel") nv = t["name"].null_value assert nv is not None assert list(t["name"][:]) == ["a", nv, "c"] assert t["name"].null_count() == 1 +def test_utf8_from_arrow_nulls_default_to_a_mask(): + pa = pytest.importorskip("pyarrow") + at = pa.table({"name": pa.array(["a", None, "c"], type=pa.string())}) + t = CTable.from_arrow(at.schema, at.to_batches()) + assert t["name"].null_storage == "mask" + assert t["name"].null_value is None + assert t["name"].is_null().tolist() == [False, True, False] + assert t["name"].null_count() == 1 + + def test_utf8_from_arrow_fixed_width_max_len(): pa = pytest.importorskip("pyarrow") at = pa.table({"name": pa.array(["hi", "there"], type=pa.string())}) @@ -1759,7 +1775,9 @@ def test_ctable_utf8_index_reopen_nulls_last(tmp_path): from dataclasses import make_dataclass path = str(tmp_path / "utf8_index.b2t") - row_cls = make_dataclass("Row", [("name", str, blosc2.field(blosc2.utf8(nullable=True)))]) + row_cls = make_dataclass( + "Row", [("name", str, blosc2.field(blosc2.utf8(nullable=True, null_storage="sentinel")))] + ) values = ["pear", "apple", None, "banana"] t = blosc2.CTable(row_cls, urlpath=path, mode="w") t.extend({"name": values}, validate=False) @@ -1817,7 +1835,7 @@ def test_ctable_utf8_index_answers_scalar_predicates(nullable, tmp_path): values = ["pear", "apple", "café", "banana", "apple", "pear"] if nullable: values = [*values, None] - spec = blosc2.utf8(nullable=True) if nullable else blosc2.utf8() + spec = blosc2.utf8(nullable=True, null_storage="sentinel") if nullable else blosc2.utf8() row_cls = make_dataclass("Row", [("c", str, blosc2.field(spec))]) masks = {} @@ -1902,7 +1920,9 @@ def test_ctable_utf8_index_ne_on_all_null_column(tmp_path): """ from dataclasses import make_dataclass - row_cls = make_dataclass("Row", [("c", str, blosc2.field(blosc2.utf8(nullable=True)))]) + row_cls = make_dataclass( + "Row", [("c", str, blosc2.field(blosc2.utf8(nullable=True, null_storage="sentinel")))] + ) t = blosc2.CTable(row_cls, urlpath=str(tmp_path / "t.b2t"), mode="w") t.extend({"c": [None] * 6}, validate=False) t._flush_varlen_columns() From 89a7c923eced785521a56db6f0884e9ed6699604 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 18:12:05 +0200 Subject: [PATCH 11/47] Summarise indexes over the rows that carry a value (mask-based-nulls 10) An index summarises the column's physical array, so every nullable column looked to it like a column of data: a sentinel entered the per-segment extrema, and so did a mask column's fill. That is why min()/max() declined the summary shortcut for any nullable column except a NaN-sentinel float, whose nulls the builder already dropped as NaNs. The summary builder now takes a per-column validity provider -- one callable, answering from the sentinel for one storage and from the .notnull sidecar for the other -- and takes its extrema over the valid rows only. A segment with no valid row is flagged FLAG_ALL_NULL, set together with FLAG_ALL_NAN so that a reader knowing only the NaN flags still skips it. The descriptor records null_aware; an index built before this carries no such key, keeps the old bail, and is promoted by rebuild_index(). min()/max() then answer from the index for a nullable column of either storage -- 236x on a 20M-row int64. This is not a mask feature: the suite's canonical fallback case was an INT64_MIN-sentinel column, on the grounds that its sentinel *is* the block minimum, and it takes the shortcut now. Phase 5's whole-column bail for mask floats narrows to what it was really about: FLAG_HAS_NAN over the valid rows marks a genuine NaN, so such a column declines only when it holds one, which is where the scan poisons to NaN. Two things planned for this phase were deliberately not done, and the plan records why. null_order is not written: no index kind reorders nulls, so "last" would have been a promise nothing keeps -- the contract lives in _build_lex_keys. The build token is not bumped: an older index is conservative rather than wrong, and a bump would rebuild every stored index in the wild to buy a shortcut. Plus the indexed-OR lift, which the phase did not anticipate. The earlier bail generalised from "an ordered index never evaluates the predicate" to "OR over a nullable indexed column must scan", but the segment path is not ordered: it prunes blocks by their summaries and then runs the predicate through miniexpr, which has been null-aware per leaf since phase 1. Its global post-filter was not merely wrong for OR, it was unnecessary. That path now serves OR (1.61x on a 20M-row probe); the exact-position paths still bail. The OR test also parses the AST rather than searching for "|", which used to take the index away from `name == 'a|b'`. One cost, recorded rather than hidden: the per-block summaries folded during writes carry no validity, so a nullable column holding a null cannot use them and pays one decompression pass at close() -- 1.8 ms to 33.2 ms for a 20M-row int64. A nullable column with no nulls keeps that fast path untouched. Threading validity into _ColumnSummaryAccumulator is a named follow-up. Co-Authored-By: Claude Opus 5 --- RELEASE_NOTES.md | 25 ++ doc/reference/ctable.rst | 21 ++ plans/mask-based-nulls.md | 113 ++++++- src/blosc2/ctable.py | 116 ++++--- src/blosc2/ctable_indexing.py | 126 +++++++- src/blosc2/indexing.py | 170 ++++++++-- tests/ctable/test_column.py | 13 +- tests/ctable/test_null_aware_indexes.py | 350 +++++++++++++++++++++ tests/ctable/test_null_mask_expressions.py | 36 ++- 9 files changed, 871 insertions(+), 99 deletions(-) create mode 100644 tests/ctable/test_null_aware_indexes.py diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d48ddf464..51473630a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -58,6 +58,31 @@ One deliberate semantic difference: in a mask column `NaN` is a **value**, following Arrow, and only the sidecar marks a null. Sentinel float columns keep NaN-as-null. See "Where nulls are stored" in the CTable reference. +#### Column indexes are null-aware + +Every index kind stores per-segment `min`/`max`, and those extrema are now taken +over the rows that carry a **value**: a column's nulls are read from its +validity channel and left out, and a segment with no value at all is flagged +rather than summarised. This applies to both storages — an `INT64_MIN` sentinel +is exactly as invisible to a summary as a mask column's fill. + +Two things follow. `Column.min`/`Column.max` answer from the index for a +nullable column (**236x** on a 20M-row `int64`, measured) where before every +nullable column but a NaN-sentinel float had to scan; and `where()` with an `OR` +over a nullable indexed column uses the index instead of falling back to a full +scan (**1.6x** on a 20M-row two-column probe). The `OR` fallback existed because +the only null filtering available was global, and a global filter drops a row +that is null in one branch but matches the other; the segment path never needed +it, because it *evaluates* the predicate, which has been null-aware per leaf +since the string-predicate fix below. + +Indexes written by an earlier release are read as not null-aware and keep the +old fallback, so nothing silently changes meaning; `rebuild_index()` promotes +them. Building an index over a nullable column that actually holds nulls now +costs one decompression pass (33 ms for a 20M-row `int64` column) because the +incremental per-block summaries folded during writes carry no validity; a +nullable column with no nulls keeps that fast path untouched. + ### Bug fixes - **`group_by` returned the wrong `min` for a `bool` value column.** The diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index cfb559ebf..68937e382 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -176,6 +176,10 @@ One semantic difference is deliberate: in a **mask** column ``NaN`` is a column keeps NaN-as-null. So ``dropna``, ``group_by`` and ``min``/``max`` can differ between the two for float columns holding a real NaN. +Indexes understand both storages: a column index summarises only the rows that +carry a value, so ``min``/``max`` and ``where`` are as fast on a nullable +column as on a plain one. See `Indexes`_. + Converting between them ~~~~~~~~~~~~~~~~~~~~~~~ @@ -697,6 +701,23 @@ Choosing an index kind than 50 % of candidate segments, the planner skips the index and falls back to a full scan to avoid per‑segment evaluation overhead. +Indexes on nullable columns + Every index kind stores per‑segment ``min``/``max``, and since 4.10.2 those + extrema are taken over the rows that carry a **value**: a column's nulls are + read from its validity channel — the ``.notnull`` sidecar of a mask column, + the reserved value of a sentinel one — and left out. A segment with no + value at all is flagged rather than summarised. + + Two things follow. :meth:`Column.min` and :meth:`Column.max` answer from + the summaries for a nullable column instead of scanning it (~240x on a + 20M‑row column), where before any nullable column except a NaN‑sentinel + float had to fall back; and segment pruning is tighter, because a block + whose only large values are nulls no longer looks like a candidate. + + Indexes built by an earlier release are read as *not* null‑aware and keep + the old fallback, so nothing silently changes meaning; + :meth:`CTable.rebuild_index` promotes them. + .. autosummary:: CTable.create_index diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index c2fcc33a5..7ed41b27c 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1,9 +1,10 @@ # Mask-based nullable columns for CTable -> **Status: Phases 0–9 landed 2026-08-08; only Phase 10 remains.** Lossless Arrow/Parquet +> **Status: complete — Phases 0–10 landed 2026-08-08.** Lossless Arrow/Parquet > round-trip works for every V1 kind, sort/groupby/query honour a sidecar, `convert_nulls` migrates -> columns in either direction, and **mask storage is now the default** — a bare `nullable=True` -> resolves to it. Six premises were disproven during +> columns in either direction, **mask storage is now the default** — a bare `nullable=True` +> resolves to it — and column indexes summarise only the rows that carry a value, so `min`/`max` +> and indexed `OR` no longer fall back on a nullable column. Six premises were disproven during > implementation and are corrected in place, each in a blockquote beside the text it corrects: the > index path cannot be fixed by a null-aware expression (§Expression layer), the bool dtype-flip > cannot move out of `__init__` (§Schema layer), ndarray columns do not get lazy null propagation @@ -13,7 +14,13 @@ > mask half of the query path undone**, and nobody noticed until sort work went looking > (§Expression layer, "Addendum 2") — and Phase 8 an eighth: the in-place migration ordering > recorded below is **wrong at its middle step**, and moving that step last makes every -> intermediate state correct rather than merely recoverable (§Migration). Drafted 2026-08-08. +> intermediate state correct rather than merely recoverable (§Migration). Phase 10 added a ninth, +> which is a partial retraction of the fourth: the summary fast path is unsound only for a *genuine* +> NaN, not for a whole mask float column, and `null_order` — planned beside `null_aware` — records +> a promise no index kind keeps (§Reductions, §Sort and indexes). It also narrows the first: an +> ordered index cannot be fixed by a null-aware expression, but the *segment* index was never +> ordered and needed no fixing, which is what lets indexed `OR` work +> (§Expression layer, "Addendum 3"). Drafted 2026-08-08. > Revised 2026-08-08 after review: lazy sidecar materialization (decision 9), `NullPolicy` > inference instead of raising, staged default flip (Phase 9), null-predicate rewrite pulled > forward to Phase 1, sidecar suffix renamed `.notnull`. @@ -525,6 +532,41 @@ never consults a side channel. Two honest routes: Until (2) lands, mask columns take the same bail as sentinel columns. **Do not ship a fast path that is silently wrong.** +> **As built (2026-08-08, Phase 10).** Route 2, and it lands in `indexing.py` rather than +> `ctable_indexing.py` — the summary builder lives there; what `ctable_indexing.py` contributes is +> the validity channel to build with. Five notes: +> +> - **The builder takes a callable, not an array.** `validity(values, start, stop) -> valid | None` +> is threaded through `_build_levels_descriptor{,_ooc}` and called per chunk. That shape is what +> lets a *sentinel* column answer for free from the values the builder already decompressed +> (`~sentinel_mask(values, nv)`) while a mask column reads one byte per row off its sidecar — +> the same split every other phase made, arriving here unchanged. +> - **`FLAG_ALL_NULL` is set together with `FLAG_ALL_NAN`, deliberately.** The established meaning +> of `FLAG_ALL_NAN` at every consumer is "the extrema are placeholders, skip this segment", which +> is exactly what an all-null segment needs; riding along with it means the pruning path +> (`_candidate_units_from_summary`) and the min/max path both got it right with no edit. +> - **The prediction that this is not a mask feature is confirmed, and pinned by a test that was +> already there.** `test_minmax_matches_reference`'s `k` column — an `INT64_MIN`-sentinel `int64` +> — was the suite's canonical *fallback* case, on the grounds that the sentinel **is** the block +> minimum. It takes the shortcut now, and the parametrization flipped from `False` to `True`. +> - **`FLAG_HAS_NAN` computed over the valid rows is what makes decision 6 expressible.** Phase 5 +> disabled the shortcut for the whole mask-float column because a NaN fill and a NaN value were +> indistinguishable to the summary. They are distinguishable to a *null-aware* summary: the flag +> marks only a NaN among valid rows, so a mask float column qualifies as a source and declines +> only when it actually holds a NaN — where the scan poisons to NaN and the summaries must not +> contradict it. `test_summary_minmax_shortcut_stays_disabled_for_mask_columns` was rewritten in +> place as `test_a_genuine_nan_still_keeps_a_mask_float_column_off_the_shortcut`, with its +> converse beside it. +> - **Measured: 236.89x** for `min()` on a 20M-row nullable `int64` (39.32 ms → 0.17 ms), which is +> the same order as the ~240x the non-nullable path already claimed. The cost is on the build +> side: the per-block summaries folded incrementally during writes carry no validity, so a +> nullable column holding a null cannot use them and pays one decompression pass at `close()` +> (1.8 ms → 33.2 ms for that column). A nullable column with **no** nulls keeps the fast path +> untouched — it needs no validity provider, and is marked `null_aware` anyway. Threading +> validity into `_ColumnSummaryAccumulator` is the named follow-up; it was left out because it +> reaches into `extend`'s write loop and one of its two feed sites (the Arrow writer's +> `on_write=` callback) has no validity to give. + ### Sort and indexes `_build_lex_keys` (`:11653-11730`): the null-indicator key becomes `(~valid[live_pos]).astype(np.intp)` — @@ -546,6 +588,19 @@ Index descriptors gain `{"null_aware": true, "null_order": "last"}` (anticipated `plans/ctable-nulls.md:614-623`, present in neither `plans/ctable-indexes-opsi.md` nor the code). Read with `.get("null_aware", False)`; bump the build token so stale indexes rebuild. +> **As built (2026-08-08, Phase 10): `null_aware` yes, `null_order` no, and no token bump.** +> `null_order` would have recorded something untrue. No index kind *reorders* nulls: a FULL index +> sorts them wherever their sentinel or fill lands, which is why `_sorted_slice_positions` bails +> (Phase 7) rather than locating a null run. The nulls-last contract lives in `_build_lex_keys`, +> not in a descriptor. Recording `"null_order": "last"` beside it would have read as a promise the +> index does not keep — see the follow-up on a null-run FULL index below for what would earn it. +> +> The token bump is not needed either, and skipping it is the safer choice: `.get("null_aware", +> False)` already makes an older index take the old bail, so it is *correct* rather than stale, and +> a bump would rebuild every stored index in the wild to buy a shortcut. This is the same +> distinction Phase 7 drew for `_utf8_rank_arrays`, whose pre-mask indexes were genuinely *wrong* +> and did have to be invalidated. `rebuild_index()` promotes an old index on request. + > **As built (2026-08-08).** All four items, plus one more site and two pre-existing sort bugs that > only mask storage can reach. > @@ -685,6 +740,29 @@ Phase 1**, landing right after the `NullChannel` refactor and before any mask wo > > Verified against a NumPy SQL oracle over 32 combinations — {mask, sentinel} × {indexed, > unindexed} × 8 expression shapes including `|` and `~` — all agreeing exactly. +> +> **Addendum 3 (2026-08-08, Phase 10): the indexed-OR bail is lifted for the path that evaluates, +> and kept for the paths that do not.** The correction above is right that a null-aware expression +> cannot fix an index which never evaluates it — but it over-generalized from there to "OR over a +> nullable indexed column must fall back to the scan". Only *some* index paths answer without +> evaluating. The segment/candidate-unit path prunes blocks by their summaries and then runs the +> predicate through miniexpr over the survivors, so its result is exact and +> `_exclude_null_positions` there was not merely wrong for OR — it was unnecessary. That path now +> serves OR with the filter skipped. The exact-position paths (FULL/PARTIAL/BUCKET), which answer +> by slicing the sorted column, still bail. +> +> Two details: +> +> - **Those three bails are unreachable today, and are kept as a guard.** `_plan_exact_conjunction` +> declines any expression containing an OR, so an exact plan cannot coexist with one. They exist +> so a planner that learns to build one cannot silently reintroduce the bug. +> - **The OR test now parses.** `"|" in expression` also fires on a string literal that contains +> one (`name == 'a|b'`), which took the index away from a query with no OR in it at all; +> `_expression_has_or` walks the AST for `ast.Or`/`ast.BitOr` instead. +> +> Measured on a 20M-row two-column probe, `(a > 4000) | (b > 6000)` with SUMMARY indexes on both: +> **1.61x** (12.35 ms → 7.65 ms). Modest, and honestly so — the miniexpr scan it was falling back +> to is already fast; the win is proportional to how much the summaries prune. ### Groupby @@ -857,6 +935,13 @@ New files: preserves storage under a mask-default policy. - A version-gate test: hand-build a `version: 3` schema dict and assert a simulated old accept-list `(1, 2)` raises a clear `ValueError` naming the version. +- `tests/ctable/test_null_aware_indexes.py` (Phase 10): the summaries at unit level (extrema over + the valid rows, `FLAG_ALL_NULL`, `FLAG_HAS_NAN` marking a value and not a fill); the `null_aware` + claim, including the null-free mask column that earns it without a sidecar and the older index + that must not be trusted with it; `min`/`max` against the scan for every V1 kind and both + storages, plus full-range `int8`, the straddling tail block, an all-null column, and a null + written *after* the build; and indexed `OR`, asserted to use the index rather than only to be + right. End-to-end smoke, run manually: import a nullable-bool + full-range-`int8` + free-text-utf8 Parquet file, round-trip it, and assert `pq.read_table(out).equals(pq.read_table(in))` — the case that is @@ -891,12 +976,16 @@ default-created tables require them. | 7 | ✅ **Sort + groupby.** `_build_lex_keys` (the indicator key is nulls-last *entirely*, not a refinement), `_sorted_positions_from_full_index` (3.0x for `U16`, 1.2x for `int64` — the I/O win is in bytes, not proportionally in time), `_utf8_rank_arrays(valid=)` plus a `null_aware` staleness rule no O(1) signal could replace, `_sorted_slice_positions` bails, groupby `_null_mask(valid=)` **plus `_CodedKeyChunk`**, which this section had not anticipated: a mask *key* column needs a reserved null code, not a threaded flag. **Plus the mask half of Phase 1**, which had never been done — `where()` leaked nulls on both the scan and the index (see §Expression layer, Addendum 2). Three pre-existing storage-independent bugs fixed on the way: descending sort of `bool` (raised) and of full-range signed ints (wrong order), and groupby `min`/`max` over `bool` (always `False`). `tests/ctable/test_null_mask_sort_groupby.py` (75 tests). | **L** (was M) | Med-High | | 8 | ✅ **Migration + docs.** `convert_nulls` both directions for every V1 kind, refusing what a sentinel cannot represent; the crash ordering **corrected** (fill after the schema flip, not before) and asserted; a persistent in-place dtype change refused with a reason; `_detach_schema` so a converted copy stops relabelling its source; the table-level `info` null tag (`Column.null_storage` and the per-column `info` rows already existed). `doc/reference/ctable.rst` gains a "Where nulls are stored" section and a rewritten null-policy resolution order; release notes. Three storage-independent bugs fixed on the way: `copy()`'s off-by-one write watermark (which *raised* in `add_column`), and a nullable `uint8` ndarray column coming back as `bool_`. `tests/ctable/test_null_migration.py` (50 tests). | M (was S–M) | Low | | 9 | ✅ **Default flips to `"mask"`.** Not a one-line change: `null_storage` had to become tri-state (`None` = unspecified) so that a type-wide sentinel field can still imply sentinel storage without contradicting the new default, and ~65 tests that wrote a sentinel literally to mean "null" had to say `null_storage="sentinel"` and mean it. Three real gaps the flip exposed, all fixed: **CSV import/export was sentinel-only** (`from_csv` raised on an empty field, `to_csv` wrote the fill as data), **`~` on a mask bool column selected its nulls**, and **the Arrow importer ignored the type-wide sentinel inference**, so `NullPolicy(signed_int_strategy="max")` meant one thing for a declared schema and another for an inferred one. Landed in the **same** session as Phase 6, not a release later — see the note below. | M (was S) | Med (was Low) | -| 10 | **Index null-awareness remainder** *(independent)*. Mask-aware summary builder; `null_aware`/`null_order` descriptors; re-enable `_summary_minmax_source` for mask and sentinel columns alike. | **L** | High | +| 10 | ✅ **Index null-awareness remainder** *(independent)*. Summary builder takes a per-column validity provider (one callable, both storages); `FLAG_ALL_NULL` riding on `FLAG_ALL_NAN`; `null_aware` in the descriptor, **`null_order` deliberately not recorded** and **no token bump** — see the corrections above. `_summary_minmax_source` re-enabled for mask *and* sentinel columns (**236.89x** on a 20M-row `int64` `min()`), with `FLAG_HAS_NAN` over the valid rows narrowing Phase 5's whole-column bail down to the one genuine-NaN case it was really about. Plus the **indexed-OR lift**, which this row did not anticipate: the segment path evaluates the predicate, so it never needed the global post-filter that forced the bail (1.61x on a 20M-row probe). One cost, recorded: an index over a nullable column holding nulls can no longer use the incremental write-time summaries and pays a decompression pass at `close()`. `tests/ctable/test_null_aware_indexes.py` (22 tests). | M (was L) | Med (was High) | The riskiest, most-coupled work is isolated into Phases 4, 7 and 10, each of which can slip without blocking the others. Phase 9 is a policy change, not code — its only prerequisite is that Phases 2–8 have soaked for a release. +Phase 10 came in smaller than budgeted for the same reason Phase 5 did: Phases 0–7 had already +made every consumer read nullity through one channel, so the remaining work was to *supply* that +channel to one more builder rather than to teach a new subsystem about nulls. + > **Deviation from decision 1 (2026-08-08), recorded deliberately.** Phase 9 landed in the same > session as Phase 6, not a release later. The staging existed so that version-3-capable readers > would be in circulation before default-created tables required them; shipping both at once means @@ -919,5 +1008,15 @@ that Phases 2–8 have soaked for a release. - **`__setitem__`'s fast paths** for mask columns (deferred in Phase 4, still unmeasured). - **Kleene three-valued logic**, decision 8 — which now also owns the operator-form negation leak pinned as a `strict=True` xfail in `tests/ctable/test_null_predicate_rewrite.py`. -- **Phase 10** as listed above: mask-aware summary builder, `null_aware`/`null_order` descriptors, - and with them a genuinely indexed `OR` over a nullable column. +- **Validity through `_ColumnSummaryAccumulator`** (Phase 10). The per-block summaries folded + during writes carry no validity, so a nullable column holding a null cannot use them and pays a + decompression pass at `close()` — 1.8 ms → 33.2 ms for a 20M-row `int64`. `extend`'s feed site + already has the validity in hand (`batch_valid`); the Arrow writer's `on_write=` callback does + not, so the accumulator would need a "nullable columns must be fed validity or invalidate" rule + and that path would keep today's cost. +- **A null-run FULL index**, which is what would earn a truthful `"null_order": "last"`. Sorting + nulls last *within ties* makes them a contiguous range of the sorted array, so a range query + could subtract it in sorted space — exact, per leaf, no I/O. That would let + `_exclude_null_positions` go away entirely, put `OR` on the exact-position paths, and give + `_sorted_slice_positions` (which bails since Phase 7) its window read back. It needs the sort key + and the external-merge builder to carry validity, in every index kind. diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index b88bf313e..bc0026a3b 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -3218,15 +3218,16 @@ def _lazy_aggregate_fastpath(self, op: str, *, where=None, dtype=None, ddof: int return NotImplemented def _summary_minmax_source(self): - """Return ``(sidecar_path, dtype, nullable, segment_len)`` for a - summary-readable ``min``/``max``, or ``None`` when the index shortcut is - not provably correct. + """Return ``(sidecar_path, dtype, nullable, nan_is_data, segment_len)`` + for a summary-readable ``min``/``max``, or ``None`` when the index + shortcut is not provably correct. Excluded: a view (its summary describes the base table); a column kind - without numeric/string block extrema; a leaky null sentinel (only a - non-nullable column, or a NaN-sentinel float — whose NaNs the summary - drops — match the nulls-skipped contract of ``min()``); and a stale, - absent, or in-memory-only index. + without numeric/string block extrema; a nullable column whose index was + built without a validity channel — its per-segment extrema cover the + sentinel or the fill as if it were data, and only a NaN-sentinel float + (whose nulls the summary drops as NaNs anyway) escapes that; and a + stale, absent, or in-memory-only index. Appends mark the index stale, so they are covered. Deletions are *not* (``delete()`` tombstones in place and leaves the index usable for @@ -3259,24 +3260,26 @@ def _summary_minmax_source(self): nullable = getattr(spec, "nullable", False) null_value = getattr(spec, "null_value", None) is_nan_float = dtype.kind == "f" and is_nan_sentinel(null_value) - if nullable and not is_nan_float: - # A non-NaN sentinel leaks into the block extrema. - # - # It is tempting to let mask-backed float columns through here too, - # on the grounds that their fill is NaN and the summary builder - # drops NaNs. That is wrong, and precisely *because* NaN is a value - # in a mask column (Arrow semantics): a genuine NaN poisons the - # scanned min()/max() to NaN, while the summary silently drops it - # and answers with a real extremum. Measured on - # ``[1.0, nan, 5.0, null, 3.0]``: scan gives nan, summaries would - # give 1.0/5.0. Making the index null-aware is the real fix - # (plans/mask-based-nulls.md, phase 10); until then mask columns - # take the same bail as sentinel ones. - return None root = table._root_table desc = root._get_index_catalog().get(self._col_name) if not desc or desc.get("stale", False): return None + if nullable and not is_nan_float and not desc.get("null_aware", False): + # Both a non-NaN sentinel and a mask column's fill leak into the + # block extrema of an index built without a validity channel — the + # summary builder reads the physical array and cannot tell either + # from data. ``null_aware`` says the extrema were taken over the + # valid rows only, which is what makes them agree with a scan that + # skips nulls. Indexes built before Phase 10 carry no such key, so + # they keep the old bail and rebuild into the shortcut. + return None + # Whether a NaN in this column is a *value* rather than a null. It is + # for a non-nullable float and for a mask column (Arrow semantics, + # decision 6 of plans/mask-based-nulls.md), and it is not for a + # NaN-sentinel float. Where NaN is data it poisons a scanned + # min()/max() to NaN while the summaries drop it, so the caller must + # decline rather than answer with a real extremum. + nan_is_data = dtype.kind == "f" and not is_nan_float # A tombstoned row still sits in its block and still contributes to that # block's extrema, and the summaries index physical slots while min() # reads logical rows. Both only line up while every slot below the @@ -3293,7 +3296,38 @@ def _summary_minmax_source(self): segment_len = levels[level].get("segment_len") if not segment_len: return None - return path, dtype, nullable, int(segment_len) + return path, dtype, nullable, nan_is_data, int(segment_len) + + def _summary_minmax_tail(self, tail_start, n_live, dtype, nullable, nan_is_data, op): + """Reduce by hand the one block the summaries could not cover. + + Its tail is capacity padding, so it has no usable summary entry. The + rescan has to apply exactly the rules the summaries applied: skip the + nulls, and decline where a NaN is data and would poison the answer the + summaries gave. + + Returns the block's extremum, ``None`` when it holds nothing usable, or + ``NotImplemented`` to abandon the shortcut altogether. + """ + try: + seg = np.asarray(self[tail_start:n_live]) + except Exception: + return NotImplemented + if nullable: + null = self._nulls.null_mask_slice(seg, tail_start, n_live) + if null is not None: + seg = seg[~null] + if dtype.kind == "f" and seg.shape[0]: + nan = np.isnan(seg) + if nan.any(): + if nan_is_data: + return NotImplemented + seg = seg[~nan] + if not seg.shape[0]: + return None + if dtype.kind in "US": + return min(seg) if op == "min" else max(seg) + return seg.min() if op == "min" else seg.max() def _index_summary_minmax(self, op: str): """Exact ``min``/``max`` from the column index's block summaries, or @@ -3313,7 +3347,7 @@ def _index_summary_minmax(self, op: str): source = self._summary_minmax_source() if source is None: return NotImplemented - path, dtype, nullable, segment_len = source + path, dtype, nullable, nan_is_data, segment_len = source n_live = self._table._root_table._n_rows if n_live is None or n_live == 0: return NotImplemented @@ -3335,30 +3369,26 @@ def _index_summary_minmax(self, op: str): # Drop the padded tail: keep only blocks lying wholly below n_rows. flags = flags[:n_full] vals = vals[:n_full] - # A non-nullable float with NaN *data* makes numpy min/max return NaN, + # A float column where NaN is *data* makes numpy min/max return NaN, # but the summary dropped those NaNs — they would disagree, so bail. - if dtype.kind == "f" and not nullable and bool((flags & (FLAG_HAS_NAN | FLAG_ALL_NAN)).any()): + # On a null-aware index FLAG_HAS_NAN is raised over the valid rows + # only, so a mask column's NaN fill does not trip this; a genuine NaN + # value does, which is exactly the distinction decision 6 draws. + if nan_is_data and bool((flags & FLAG_HAS_NAN).any()): return NotImplemented - valid = (flags & FLAG_ALL_NAN) == 0 + valid = (flags & FLAG_ALL_NAN) == 0 # also clear on all-null segments vals = vals[valid] # The straddling block is not summarisable (its tail is padding), so read # just its live rows. This is also the whole answer when the column is # shorter than one block, in which case no summary entry is usable. - tail = n_live - n_full * segment_len - if tail: - try: - seg = np.asarray(self[n_full * segment_len : n_live]) - except Exception: + if n_live - n_full * segment_len: + seg_val = self._summary_minmax_tail( + n_full * segment_len, n_live, dtype, nullable, nan_is_data, op + ) + if seg_val is NotImplemented: return NotImplemented - if dtype.kind == "f": - seg = seg[~np.isnan(seg)] - if not nullable and seg.shape[0] != tail: - return NotImplemented # NaN data: see above - if seg.shape[0]: - seg_val = min(seg) if dtype.kind in "US" else seg.min() - if op == "max": - seg_val = max(seg) if dtype.kind in "US" else seg.max() + if seg_val is not None: vals = np.concatenate([vals, np.asarray([seg_val], dtype=dtype)]) if vals.shape[0] == 0: @@ -10894,7 +10924,13 @@ def _invalidate_all_summary_accumulators(self) -> None: def _precomputed_summary_for(self, name: str): """Return ``{"block": summaries}`` for *name* if a valid accumulator - fully covers the column's physical extent, else None.""" + fully covers the column's physical extent, else None. + + These are folded as rows are written, with no validity in hand, so the + index builder ignores them for a column that has nulls and decompresses + instead (see ``_build_levels_descriptor_ooc``). A nullable column with + no nulls keeps the fast path. + """ accs = self.__dict__.get("_summary_accumulators") if not accs: return None diff --git a/src/blosc2/ctable_indexing.py b/src/blosc2/ctable_indexing.py index e4b359ccf..6ba285dea 100644 --- a/src/blosc2/ctable_indexing.py +++ b/src/blosc2/ctable_indexing.py @@ -182,6 +182,26 @@ def _persist_utf8_vocab(full: dict, meta: dict, sorted_vocab: np.ndarray) -> Non meta["vocab_path"] = vocab_path +def _expression_has_or(expression: str) -> bool: + """Whether *expression* really contains a boolean OR. + + A substring test for ``"|"`` also fires on a string literal that happens to + contain one (``name == 'a|b'``), and the answer decides whether a nullable + indexed column may keep its index -- so it is worth parsing for. + Unparseable input answers True, which is the conservative side. + """ + try: + tree = ast.parse(expression, mode="eval") + except SyntaxError: + return True + for node in ast.walk(tree): + if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.Or): + return True + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): + return True + return False + + class _DictRankWrapper: """Wrap a dictionary column's codes NDArray, translating codes to alphabetical ranks on read. @@ -526,6 +546,54 @@ def _resolve_index_catalog_entry( return matches[0] raise TypeError("must specify col_name, expression, or name") + def _index_validity_provider(self, col_name: str): + """``(provider, null_aware)`` for *col_name*'s summary build. + + *provider* is a ``(values, start, stop) -> valid`` callable, or ``None`` + when every row is valid. It is what makes an index null-aware: the + segment-summary builder calls it per chunk and keeps null rows out of + the per-segment extrema. Without it a nullable column's summaries + describe its fill (or its sentinel) as if it were data, which is why + the ``min()``/``max()`` shortcut had to decline them. + + *null_aware* is the claim recorded in the descriptor, and it is true in + one case where *provider* is ``None``: a mask column that has never + held a null (no sidecar, per decision 9 of plans/mask-based-nulls.md). + Its summaries agree with a null-skipping scan for free, so it should + not pay the bail. + """ + col = self._schema.columns_by_name.get(col_name) + spec = col.spec if col is not None else None + if spec is None: + return None, False + kind = kind_of_spec(spec) + if kind == NULL_MASK: + mask = self._null_mask(col_name) + if mask is None: + return None, True + + def _valid_from_sidecar(values, start, stop, _mask=mask): + # One byte per row off the sidecar, and the values are not + # consulted at all. + valid = np.asarray(_mask[start:stop], dtype=bool) + if valid.shape[0] < stop - start: + # Capacity the sidecar has not been grown over yet: those + # rows hold no data, and _grow() writes True over new tail + # rows, so "valid" is the consistent answer for them. + pad = np.ones(stop - start - valid.shape[0], dtype=bool) + valid = np.concatenate([valid, pad]) + return valid + + return _valid_from_sidecar, True + if kind == NULL_SENTINEL: + null_value = spec.null_value + + def _valid_from_sentinel(values, start, stop, _nv=null_value): + return ~sentinel_mask(values, _nv) + + return _valid_from_sentinel, True + return None, False + def _build_index_persistent( self, col_name: str, @@ -541,6 +609,7 @@ def _build_index_persistent( opsi_max_cycles: int | None = None, summary_levels: tuple[str, ...] | None = None, precomputed_summaries: dict | None = None, + validity=None, ) -> dict: """Build index sidecar files for a persistent-table column; return the descriptor.""" import tempfile @@ -593,6 +662,7 @@ def _build_index_persistent( cparams_obj, summary_levels=summary_levels, precomputed_summaries=precomputed_summaries, + validity=validity, ) bucket = ( _build_bucket_descriptor_ooc( @@ -636,6 +706,7 @@ def _build_index_persistent( full, cparams_obj, opsi, + validity is not None, ) else: values = _values_for_target(proxy, target) @@ -649,6 +720,7 @@ def _build_index_persistent( persistent, cparams_obj, summary_levels=summary_levels, + validity=validity, ) bucket = ( _build_bucket_descriptor(proxy, token, kind, values, optlevel, persistent, cparams_obj) @@ -685,6 +757,7 @@ def _build_index_persistent( full, cparams_obj, opsi, + validity is not None, ) result = _copy_descriptor(descriptor) @@ -938,6 +1011,15 @@ def create_index( # noqa: C901 col_arr = _DictRankWrapper( dict_col.codes, code_to_rank, null_rank, null_code, dict_col.spec.nullable, n_phys ) + # utf8 and dictionary columns are indexed by *rank*, and a null already + # has a rank of its own there (``null_rank``, sorting last), so their + # summaries are null-aware by construction and there is nothing for a + # validity channel to add -- nor would it line up, the ranks array being + # a different array from the column's payload. + validity, null_aware = ( + (None, False) if (is_utf8 or is_dictionary) else self._index_validity_provider(col_name) + ) + is_persistent = self._storage.index_anchor_path(col_name) is not None if is_persistent: @@ -954,6 +1036,7 @@ def create_index( # noqa: C901 opsi_max_cycles=opsi_max_cycles, summary_levels=summary_levels, precomputed_summaries=precomputed_summaries if kind_str == "summary" else None, + validity=validity, ) else: # In-memory path: materialise ranks as a proper NDArray (small tables only). @@ -976,6 +1059,7 @@ def create_index( # noqa: C901 opsi_max_cycles=opsi_max_cycles, summary_levels=summary_levels, precomputed_summaries=precomputed_summaries if kind_str == "summary" else None, + validity=validity, ) store = _IN_MEMORY_INDEXES[id(col_arr)] descriptor = _copy_descriptor(store["indexes"]["__self__"]) @@ -989,6 +1073,10 @@ def create_index( # noqa: C901 _persist_utf8_vocab(full, utf8_rank_meta, utf8_vocab) full["utf8_rank"] = utf8_rank_meta + # A null-free mask column builds with no validity provider (there is + # nothing to exclude) yet its summaries are null-aware all the same. + descriptor["null_aware"] = bool(descriptor.get("null_aware") or null_aware) + value_epoch, _ = self._storage.get_epoch_counters() descriptor["built_value_epoch"] = value_epoch @@ -1494,12 +1582,26 @@ def _try_index_where(self, expr_result: blosc2.LazyExpr) -> np.ndarray | None: # Global null post-filtering is not correct for OR expressions: it would # drop a row that is null in one column but matches the other branch. - # (The per-leaf rewrite makes the *expression* handle OR correctly; it - # cannot fix an index that never evaluates the expression, so an OR over - # a nullable indexed column still falls back to the scan -- which is now - # itself null-aware, and so now returns the right answer.) - if nullable_indexed and ("|" in expr_result.expression or " or " in expr_result.expression): - return None + # Which paths that rules out depends on *how* each one answers. + # + # The segment/candidate-unit path evaluates the predicate through + # miniexpr over the surviving blocks, and the predicate is null-aware + # per leaf by the time it gets here (CTable._rewrite_null_predicates for + # the string form, Column._null_aware_compare for the operator one), so + # its result is already exact and the post-filter is not merely + # incorrect for OR but unnecessary. That path therefore serves OR, with + # the filter skipped. + # + # The exact-position paths (FULL/PARTIAL/BUCKET) answer by taking an + # ordered *range* of the sorted column and never evaluate anything, so + # the post-filter is load-bearing there and an OR over a nullable + # indexed column still falls back to the scan -- which is itself + # null-aware, and so returns the right answer. Those bails are + # belt-and-braces today: the planner declines an exact plan for any + # expression containing an OR, so such a plan cannot reach them. They + # are what stops a planner that learns to build one from silently + # reintroducing the bug. + skip_null_filter = bool(nullable_indexed) and _expression_has_or(expr_result.expression) # Inject every usable table-owned descriptor so plan_query can combine them. # In .b2z read mode all columns share the same urlpath, so _array_key() @@ -1552,9 +1654,13 @@ def _exclude_null_positions(positions): return positions if plan.exact_positions is not None: + if skip_null_filter: + return None # ordered range, no per-leaf null handling: see above return _exclude_null_positions(plan.exact_positions) if plan.partial_exact_positions is not None: + if skip_null_filter: + return None # ditto, and its refinement filter is global too # Cross-column refinement: the FULL index on one column gave us # exact positions, but the expression has additional predicates on # other columns. Refinement reads every operand column at those @@ -1610,6 +1716,8 @@ def _exclude_null_positions(positions): # Fall through to full scan if refinement fails if plan.bucket_masks is not None: + if skip_null_filter: + return None # bucket masks are precomputed, not evaluated # When bucket pruning covers all units (100 % of chunks are # candidates), the per‑chunk evaluation overhead outweighs the # benefit over a plain scan. Fall back to the scan path. @@ -1656,7 +1764,7 @@ def _exclude_null_positions(positions): # lazily. Only columns whose null sentinel can satisfy the # predicate (non-NaN) require the positions fall-back; NaN # sentinels are already excluded by the predicate itself. - if not nullable_needs_exclude: + if skip_null_filter or not nullable_needs_exclude: try: mask = expr_result.compute(_candidate_blocks=bitmap) except Exception: @@ -1665,10 +1773,10 @@ def _exclude_null_positions(positions): return mask positions = self._block_pruned_positions(expr_result, bitmap, primary_col_arr) if positions is not None: - return _exclude_null_positions(positions) + return positions if skip_null_filter else _exclude_null_positions(positions) _, positions = evaluate_segment_query( expression, merged_operands, {}, where_dict, plan, return_positions=True ) - return _exclude_null_positions(positions) + return positions if skip_null_filter else _exclude_null_positions(positions) return None diff --git a/src/blosc2/indexing.py b/src/blosc2/indexing.py index 6826cf086..f244641de 100644 --- a/src/blosc2/indexing.py +++ b/src/blosc2/indexing.py @@ -18,7 +18,7 @@ import tempfile import warnings import weakref -from collections.abc import Mapping +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict, dataclass from pathlib import Path @@ -40,6 +40,11 @@ FLAG_ALL_NAN = np.uint8(1 << 0) FLAG_HAS_NAN = np.uint8(1 << 1) +# No row in the segment carries a value: every one of them is null. Set +# together with FLAG_ALL_NAN, whose established meaning -- "min/max are +# placeholders, skip this segment" -- is exactly what a reader that predates +# null-aware summaries needs to do with it. +FLAG_ALL_NULL = np.uint8(1 << 2) SEGMENT_LEVELS_BY_KIND = { # SUMMARY stores per-segment min/max. Block granularity prunes far more @@ -1154,8 +1159,48 @@ def _boundary_dtype(dtype: np.dtype) -> np.dtype: return np.dtype([("start", dtype), ("end", dtype)]) -def _segment_summary(segment: np.ndarray, dtype: np.dtype): +def _rowwise_str_minmax(data_2d: np.ndarray, dtype: np.dtype, valid_2d: np.ndarray | None): + """Per-row min/max for ``U``/``S`` rows; numpy's ufuncs lack a loop for them. + + Rows with no valid entry get the dtype's zero and are flagged by the + caller, which is the same convention the float path uses for all-NaN rows. + """ + n = data_2d.shape[0] + mins = np.empty(n, dtype=dtype) + maxs = np.empty(n, dtype=dtype) + empty = np.zeros((), dtype=dtype)[()] + for i in range(n): + row = data_2d[i] + if valid_2d is not None: + row = row[valid_2d[i]] + if row.shape[0] == 0: + mins[i] = empty + maxs[i] = empty + continue + mn = row[0] + mx = row[0] + for v in row[1:]: + if v < mn: + mn = v + if v > mx: + mx = v + mins[i] = mn + maxs[i] = mx + return mins, maxs + + +def _segment_summary(segment: np.ndarray, dtype: np.dtype, valid: np.ndarray | None = None): flags = np.uint8(0) + if valid is not None: + valid = np.asarray(valid, dtype=bool) + if not valid.all(): + if not valid.any(): + # Nothing to summarise. FLAG_ALL_NAN rides along so that a + # reader which knows only the two NaN flags still skips the + # segment instead of trusting the placeholder extrema. + zero = np.zeros((), dtype=dtype)[()] + return zero, zero, np.uint8(flags | FLAG_ALL_NULL | FLAG_ALL_NAN) + segment = segment[valid] if dtype.kind == "f": valid = ~np.isnan(segment) if not np.all(valid): @@ -1178,7 +1223,9 @@ def _segment_summary(segment: np.ndarray, dtype: np.dtype): return segment.min(), segment.max(), flags -def _compute_segment_summaries(values: np.ndarray, dtype: np.dtype, segment_len: int) -> np.ndarray: +def _compute_segment_summaries( + values: np.ndarray, dtype: np.dtype, segment_len: int, valid: np.ndarray | None = None +) -> np.ndarray: nsegments = math.ceil(values.shape[0] / segment_len) summary_dtype = _summary_dtype(dtype) summaries = np.empty(nsegments, dtype=summary_dtype) @@ -1187,7 +1234,7 @@ def _compute_segment_summaries(values: np.ndarray, dtype: np.dtype, segment_len: start = idx * segment_len stop = min(start + segment_len, values.shape[0]) segment = values[start:stop] - summaries[idx] = _segment_summary(segment, dtype) + summaries[idx] = _segment_summary(segment, dtype, None if valid is None else valid[start:stop]) return summaries @@ -1196,21 +1243,35 @@ def _fill_summaries_from_2d( summaries_arr: np.ndarray, offset: int, dtype: np.dtype, + valid_2d: np.ndarray | None = None, ) -> None: - """Fill summaries_arr[offset:offset+n] from data_2d (shape n×segment_len) with vectorized ops.""" + """Fill summaries_arr[offset:offset+n] from data_2d (shape n×segment_len) with vectorized ops. + + *valid_2d*, when given, has the same shape and excludes null rows from the + extrema, so a nullable column's summaries describe only the values a query + can actually match. Rows with no valid entry at all are flagged + ``FLAG_ALL_NULL``. + """ n = data_2d.shape[0] if n == 0: return + all_null = None if valid_2d is None else ~valid_2d.any(axis=1) if dtype.kind == "f": # All-NaN blocks make np.nanmin/nanmax emit "All-NaN slice encountered"; # their results are immediately overwritten with zero below, so silence # the (purely cosmetic) RuntimeWarning. + # NaN under a null row is the *fill*, not data, so the flags are taken + # over the valid rows only: FLAG_HAS_NAN must keep meaning "a real NaN + # value lives here", which is what makes the min()/max() shortcut safe + # to disable on exactly the columns that need it. + masked = data_2d if valid_2d is None else np.where(valid_2d, data_2d, np.nan) with np.errstate(all="ignore"), warnings.catch_warnings(): warnings.filterwarnings("ignore", r"All-NaN slice encountered", RuntimeWarning) - has_nan = np.any(np.isnan(data_2d), axis=1) - all_nan = np.all(np.isnan(data_2d), axis=1) - mins = np.nanmin(data_2d, axis=1) - maxs = np.nanmax(data_2d, axis=1) + is_nan = np.isnan(masked) + has_nan = np.any(is_nan if valid_2d is None else (is_nan & valid_2d), axis=1) + all_nan = np.all(is_nan, axis=1) + mins = np.nanmin(masked, axis=1) + maxs = np.nanmax(masked, axis=1) flags = np.where(has_nan, FLAG_HAS_NAN, np.uint8(0)).astype(np.uint8) flags = np.where(all_nan, np.uint8(FLAG_ALL_NAN | FLAG_HAS_NAN), flags) zero = dtype.type(0) @@ -1220,23 +1281,25 @@ def _fill_summaries_from_2d( if dtype.kind in "US": # String dtypes: numpy ufunc 'minimum'/'maximum' lack a loop for mx: - mx = v - mins[i] = mn - maxs[i] = mx - else: + mins, maxs = _rowwise_str_minmax(data_2d, dtype, valid_2d) + elif valid_2d is None: mins = data_2d.min(axis=1) maxs = data_2d.max(axis=1) + else: + # Substitute each row's *own* extremum for its null slots: it can + # never win the opposite reduction, so one masked pass gives the + # extrema over the valid rows. Using the row rather than the + # dtype's limits keeps this working for any orderable kind -- + # including datetime64, which has no np.iinfo. An all-null row + # degenerates to the row's own min/max and is overwritten below. + mins = np.where(valid_2d, data_2d, data_2d.max(axis=1, keepdims=True)).min(axis=1) + maxs = np.where(valid_2d, data_2d, data_2d.min(axis=1, keepdims=True)).max(axis=1) flags = np.zeros(n, dtype=np.uint8) + if all_null is not None and all_null.any(): + zero = np.zeros((), dtype=dtype)[()] + mins = np.where(all_null, zero, mins) + maxs = np.where(all_null, zero, maxs) + flags = np.where(all_null, np.uint8(FLAG_ALL_NULL | FLAG_ALL_NAN), flags).astype(np.uint8) summaries_arr["min"][offset : offset + n] = mins summaries_arr["max"][offset : offset + n] = maxs summaries_arr["flags"][offset : offset + n] = flags @@ -1429,12 +1492,14 @@ def _build_levels_descriptor( persistent: bool, cparams: dict | None = None, summary_levels: tuple[str, ...] | None = None, + validity: Callable[[np.ndarray, int, int], np.ndarray | None] | None = None, ) -> dict: levels = {} levels_to_build = summary_levels if summary_levels is not None else SEGMENT_LEVELS_BY_KIND[kind] + valid = None if validity is None else validity(values, 0, int(values.shape[0])) for level in levels_to_build: segment_len = _segment_len(array, level) - summaries = _compute_segment_summaries(values, dtype, segment_len) + summaries = _compute_segment_summaries(values, dtype, segment_len, valid) sidecar = _store_array_sidecar( array, token, kind, "summary", level, summaries, persistent, cparams=cparams ) @@ -1457,6 +1522,7 @@ def _build_levels_descriptor_ooc( cparams: dict | None = None, summary_levels: tuple[str, ...] | None = None, precomputed_summaries: dict[str, np.ndarray] | None = None, + validity: Callable[[np.ndarray, int, int], np.ndarray | None] | None = None, ) -> dict: size = int(array.shape[0]) summary_dtype = _summary_dtype(dtype) @@ -1472,11 +1538,21 @@ def _build_levels_descriptor_ooc( # column back just to recompute min/max. Only trusted when every requested # level is present with the exact expected segment count and dtype; otherwise # fall through to the decompression pass below. - use_precomputed = precomputed_summaries is not None and all( - level in precomputed_summaries - and precomputed_summaries[level].dtype == summary_dtype - and len(precomputed_summaries[level]) == nsegments_total[level] - for level in levels_to_build + # A validity provider always wins: the accumulator that produced the + # precomputed summaries folded min/max as rows were written, with no + # validity in hand, so trusting them would silently hand back extrema over + # the fill values. The caller keeps the two mutually exclusive + # (CTable._precomputed_summary_for declines for a column with nulls); the + # guard here makes that a property of the builder rather than of its caller. + use_precomputed = ( + validity is None + and precomputed_summaries is not None + and all( + level in precomputed_summaries + and precomputed_summaries[level].dtype == summary_dtype + and len(precomputed_summaries[level]) == nsegments_total[level] + for level in levels_to_build + ) ) if use_precomputed: for level in levels_to_build: @@ -1496,6 +1572,7 @@ def _build_levels_descriptor_ooc( chunk_stop = min(chunk_start + chunk_len, size) chunk_values = _slice_values_for_target(array, target, chunk_start, chunk_stop) chunk_size = chunk_stop - chunk_start + chunk_valid = None if validity is None else validity(chunk_values, chunk_start, chunk_stop) for level in levels_to_build: slen = segment_lens[level] summaries_arr = all_summaries[level] @@ -1504,10 +1581,17 @@ def _build_levels_descriptor_ooc( remainder = chunk_size % slen if n_complete > 0: data_2d = chunk_values[: n_complete * slen].reshape(n_complete, slen) - _fill_summaries_from_2d(data_2d, summaries_arr, offset, dtype) + valid_2d = ( + None + if chunk_valid is None + else chunk_valid[: n_complete * slen].reshape(n_complete, slen) + ) + _fill_summaries_from_2d(data_2d, summaries_arr, offset, dtype, valid_2d) if remainder > 0: summaries_arr[offset + n_complete] = _segment_summary( - chunk_values[n_complete * slen :], dtype + chunk_values[n_complete * slen :], + dtype, + None if chunk_valid is None else chunk_valid[n_complete * slen :], ) seg_offsets[level] = offset + n_complete + 1 else: @@ -1519,8 +1603,9 @@ def _build_levels_descriptor_ooc( for idx in range(nsegments_total[level]): start = idx * slen stop = min(start + slen, size) + seg_values = _slice_values_for_target(array, target, start, stop) all_summaries[level][idx] = _segment_summary( - _slice_values_for_target(array, target, start, stop), dtype + seg_values, dtype, None if validity is None else validity(seg_values, start, stop) ) levels = {} @@ -3990,6 +4075,7 @@ def _build_descriptor( full: dict | None, cparams: dict | None = None, opsi: dict | None = None, + null_aware: bool = False, ) -> dict: return { "name": name @@ -4013,6 +4099,11 @@ def _build_descriptor( "full": full, "opsi": opsi, "cparams": _plain_index_cparams(cparams), + # True when the segment summaries were built with a validity channel, + # so their extrema cover only rows that carry a value. Absent on every + # index built before null-aware summaries existed, which is why readers + # must spell this ``.get("null_aware", False)``. + "null_aware": bool(null_aware), } @@ -4042,6 +4133,7 @@ def create_index( opsi_max_cycles_arg = kwargs.pop("opsi_max_cycles", None) summary_levels = kwargs.pop("summary_levels", None) precomputed_summaries = kwargs.pop("precomputed_summaries", None) + validity = kwargs.pop("validity", None) if kwargs: unexpected = ", ".join(sorted(kwargs)) raise TypeError(f"unexpected keyword argument(s): {unexpected}") @@ -4074,6 +4166,7 @@ def create_index( cparams, summary_levels=summary_levels, precomputed_summaries=precomputed_summaries, + validity=validity, ) bucket = ( _build_bucket_descriptor_ooc(array, target, token, kind, dtype, optlevel, persistent, cparams) @@ -4115,11 +4208,21 @@ def create_index( full, cparams, opsi, + validity is not None, ) else: values = _values_for_target(array, target) levels = _build_levels_descriptor( - array, target, token, kind, dtype, values, persistent, cparams, summary_levels=summary_levels + array, + target, + token, + kind, + dtype, + values, + persistent, + cparams, + summary_levels=summary_levels, + validity=validity, ) bucket = ( _build_bucket_descriptor(array, token, kind, values, optlevel, persistent, cparams) @@ -4156,6 +4259,7 @@ def create_index( full, cparams, opsi, + validity is not None, ) store = _load_store(array) diff --git a/tests/ctable/test_column.py b/tests/ctable/test_column.py index 8d880206e..fa9bdd1d6 100644 --- a/tests/ctable/test_column.py +++ b/tests/ctable/test_column.py @@ -528,7 +528,7 @@ class CRow: class MinMaxRow: i: int = blosc2.field(blosc2.int64()) # non-nullable int → fast path f: float = blosc2.field(blosc2.float64(null_value=float("nan"))) # NaN float → fast path - k: int = blosc2.field(blosc2.int64(null_value=INT_MIN)) # INT64_MIN sentinel → fallback + k: int = blosc2.field(blosc2.int64(null_value=INT_MIN)) # INT64_MIN sentinel → fast since null-aware s: str = blosc2.field(blosc2.string(max_length=8)) # non-nullable string → fast path @@ -560,10 +560,17 @@ def indexed_minmax(tmp_path_factory): return path, refs -@pytest.mark.parametrize(("col", "fast"), [("i", True), ("f", True), ("s", True), ("k", False)]) +@pytest.mark.parametrize(("col", "fast"), [("i", True), ("f", True), ("s", True), ("k", True)]) def test_minmax_matches_reference(indexed_minmax, col, fast): """min()/max() equal the live non-null reference, whether or not the - summary fast path is used.""" + summary fast path is used. + + ``k`` used to be the fallback case: its ``INT64_MIN`` sentinel *is* the + block minimum, so the summaries answered with the null. Null-aware + summaries (plans/mask-based-nulls.md, phase 10) take the extrema over the + valid rows only, so it now takes the shortcut like the rest — the payoff + that phase promised for sentinel columns, not only for mask ones. + """ path, refs = indexed_minmax t = blosc2.open(path, mode="r") try: diff --git a/tests/ctable/test_null_aware_indexes.py b/tests/ctable/test_null_aware_indexes.py new file mode 100644 index 000000000..38a873637 --- /dev/null +++ b/tests/ctable/test_null_aware_indexes.py @@ -0,0 +1,350 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Null-aware column indexes (Phase 10 of plans/mask-based-nulls.md). + +An index summarises the column's *physical* array, so until now every nullable +column looked to it like a column of data: a sentinel entered the per-segment +extrema, and so did a mask column's fill. That is why ``min()``/``max()`` +declined the summary shortcut for any nullable column except the one case where +the null happens to be a NaN the summary builder already dropped. + +Phase 10 hands the builder the column's validity channel. The extrema then +cover only rows that carry a value, a segment with no such row is flagged +``FLAG_ALL_NULL``, and the descriptor records ``null_aware`` so that indexes +built before this keep the old bail instead of being trusted. The payoff lands +on *both* storages: a mask column and an ``INT64_MIN``-sentinel one are equally +unreadable to a summary that does not know about nulls. + +The second half is ``where()``: an OR over a nullable indexed column used to +fall back to a full scan, because the only null filtering available was global +and a global filter drops a row that is null in one branch but matches the +other. The segment path never needed that filter -- it *evaluates* the +predicate, which has been null-aware per leaf since Phase 1 -- so it now serves +OR directly. The ordered-range paths still bail, for the reason recorded in +§Expression layer of the plan: an index that answers by slicing the sorted +column never evaluates anything. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 +from blosc2.indexing import ( + FLAG_ALL_NAN, + FLAG_ALL_NULL, + FLAG_HAS_NAN, + _compute_segment_summaries, +) + + +def one_col(values, spec, urlpath, capacity=None, name="a"): + Row = dataclasses.make_dataclass("R", [(name, spec.python_type, blosc2.field(spec))]) + t = blosc2.CTable(Row, expected_size=max(len(values), capacity or 0), urlpath=str(urlpath), mode="w") + t.extend([(v,) for v in values]) + return t + + +# --------------------------------------------------------------------------- +# The summaries themselves +# --------------------------------------------------------------------------- + + +def test_summaries_take_their_extrema_over_the_valid_rows_only(): + """The whole phase in one assertion: -1 is a null, not the minimum.""" + values = np.array([-1, 7, 3, -1, 9, 5], dtype=np.int64) + valid = np.array([False, True, True, False, True, True]) + naive = _compute_segment_summaries(values, values.dtype, 3) + aware = _compute_segment_summaries(values, values.dtype, 3, valid) + assert naive["min"].tolist() == [-1, -1] + assert aware["min"].tolist() == [3, 5] + assert aware["max"].tolist() == [7, 9] + + +def test_an_all_null_segment_is_flagged_rather_than_summarised(): + """FLAG_ALL_NAN rides along with FLAG_ALL_NULL so that a reader which knows + only the NaN flags still skips the segment instead of trusting its zeros.""" + values = np.array([1, 2, 8, 9], dtype=np.int64) + valid = np.array([True, True, False, False]) + summaries = _compute_segment_summaries(values, values.dtype, 2, valid) + assert not summaries["flags"][0] & FLAG_ALL_NULL + assert summaries["flags"][1] & FLAG_ALL_NULL + assert summaries["flags"][1] & FLAG_ALL_NAN + # The placeholder extrema must not look like data. + assert summaries["min"][1] == 0 + assert summaries["max"][1] == 0 + + +def test_has_nan_marks_a_real_nan_and_not_a_nan_fill(): + """This flag is what keeps min() honest on a mask float column. + + A mask column's fill *is* NaN, so a flag raised over all rows would mark + every segment holding a null. Taken over the valid rows it marks exactly + the segments where NaN is data -- the case decision 6 says must poison + min() to NaN, and so must not be answered from the summaries. + """ + values = np.array([1.0, np.nan, 5.0, np.nan], dtype=np.float64) + valid = np.array([True, False, True, True]) # index 1 is a null, index 3 a value + summaries = _compute_segment_summaries(values, values.dtype, 2, valid) + assert not summaries["flags"][0] & FLAG_HAS_NAN # the fill + assert summaries["flags"][1] & FLAG_HAS_NAN # the value + assert summaries["min"][0] == 1.0 + + +def test_string_segments_skip_their_nulls_too(): + values = np.array(["", "pear", "", "apple"], dtype="U8") + valid = np.array([False, True, False, True]) + summaries = _compute_segment_summaries(values, values.dtype, 4, valid) + assert summaries["min"][0] == "apple" + assert summaries["max"][0] == "pear" + + +# --------------------------------------------------------------------------- +# The descriptor claim +# --------------------------------------------------------------------------- + +STORAGES = [ + pytest.param(blosc2.int64(null_storage="mask"), False, id="mask"), + pytest.param(blosc2.int64(nullable=True, null_value=-1), True, id="sentinel"), +] + + +@pytest.mark.parametrize(("spec", "write_sentinel"), STORAGES) +def test_a_nullable_column_records_null_aware(spec, write_sentinel, tmp_path): + vals = [None if i % 7 == 0 else i for i in range(500)] + if write_sentinel: + vals = [-1 if v is None else v for v in vals] + t = one_col(vals, spec, tmp_path / "d.b2t") + t.create_index("a", kind="summary") + assert t._get_index_catalog()["a"]["null_aware"] is True + + +def test_a_null_free_mask_column_is_null_aware_without_a_sidecar(tmp_path): + """Nothing to exclude is not the same as nothing known: the summaries of a + column that has never held a null already agree with a null-skipping scan, + so it must not pay the bail.""" + t = one_col(list(range(500)), blosc2.int64(null_storage="mask"), tmp_path / "n.b2t") + assert t._null_mask("a") is None + t.create_index("a", kind="summary") + assert t._get_index_catalog()["a"]["null_aware"] is True + assert t["a"]._index_summary_minmax("min") == 0 + + +def test_an_index_built_before_phase_10_keeps_the_bail(tmp_path): + """The staleness rule for this phase. An older index carries no + ``null_aware`` key and its extrema cover the sentinel, so it must be read as + unusable rather than as False-meaning-anything-else.""" + vals = [-1 if i % 7 == 0 else i for i in range(500)] + t = one_col(vals, blosc2.int64(nullable=True, null_value=-1), tmp_path / "old.b2t") + t.create_index("a", kind="summary") + assert t["a"]._summary_minmax_source() is not None + + del t._get_index_catalog()["a"]["null_aware"] # simulate the older build + assert t["a"]._summary_minmax_source() is None + assert t["a"].min() == 1 # the scan still answers, and answers correctly + + t.rebuild_index("a") + assert t["a"]._summary_minmax_source() is not None + + +# --------------------------------------------------------------------------- +# min() / max() through the summaries +# --------------------------------------------------------------------------- + +MINMAX_KINDS = [ + pytest.param("int64", blosc2.int64(null_storage="mask"), lambda i: i * 3 % 977, id="int64"), + pytest.param( + "float64", blosc2.float64(null_storage="mask"), lambda i: (i * 7 % 991) / 3.0, id="float64" + ), + pytest.param("uint16", blosc2.uint16(null_storage="mask"), lambda i: i * 5 % 60_000, id="uint16"), + pytest.param("bool", blosc2.bool(null_storage="mask"), lambda i: bool(i % 3), id="bool"), + pytest.param( + "string", blosc2.string(max_length=8, null_storage="mask"), lambda i: f"s{i % 977:05d}", id="string" + ), +] + + +@pytest.mark.parametrize(("label", "spec", "value_of"), MINMAX_KINDS) +def test_the_shortcut_agrees_with_the_scan_for_every_kind(label, spec, value_of, tmp_path): + n = 20_000 + vals = [None if i % 11 == 0 else value_of(i) for i in range(n)] + live = [v for v in vals if v is not None] + t = one_col(vals, spec, tmp_path / f"{label}.b2t") + t.create_index("a", kind="summary") + + assert t["a"]._index_summary_minmax("min") is not NotImplemented + assert t["a"].min() == min(live) + assert t["a"].max() == max(live) + + +def test_a_full_range_int8_reports_its_true_extrema(tmp_path): + """The case no sentinel column can express, so nothing else could test it: + -128 and 127 are both data and there is no spare value to mean null.""" + n = 5_000 + vals = [None if i % 9 == 0 else (i % 256) - 128 for i in range(n)] + live = [v for v in vals if v is not None] + t = one_col(vals, blosc2.int8(null_storage="mask"), tmp_path / "i8.b2t") + t.create_index("a", kind="summary") + assert t["a"]._index_summary_minmax("min") == -128 + assert t["a"].min() == min(live) == -128 + assert t["a"].max() == max(live) == 127 + + +def test_the_sentinel_storage_gets_the_same_answer(tmp_path): + """Phase 10 is not a mask feature: an INT64_MIN sentinel is exactly as + invisible to a summary that does not consult a validity channel.""" + imin = np.iinfo(np.int64).min + n = 20_000 + vals = [imin if i % 11 == 0 else (i * 3 % 977) for i in range(n)] + live = [v for v in vals if v != imin] + t = one_col(vals, blosc2.int64(nullable=True, null_value=imin), tmp_path / "s.b2t") + t.create_index("a", kind="summary") + assert t["a"]._index_summary_minmax("min") == min(live) + assert t["a"].max() == max(live) + + +def test_nulls_in_the_straddling_block_are_skipped_by_hand(tmp_path): + """The one block the summaries cannot cover is rescanned, and that rescan + has to apply the same rule they did.""" + n = 100_003 # big enough for several whole blocks, and not a multiple of one + vals = list(range(10, n + 10)) + t = one_col(vals, blosc2.int64(null_storage="mask"), tmp_path / "tail.b2t") + t.create_index("a", kind="summary") + segment_len = t["a"]._summary_minmax_source()[-1] + tail_start = (n // segment_len) * segment_len + assert tail_start < n, "the fixture needs a partial trailing block" + + # Put both a new minimum and a null in the rescanned tail. + t["a"][tail_start] = 0 + t["a"][tail_start + 1] = None + t.rebuild_index("a") + assert t["a"]._index_summary_minmax("min") == 0 + assert t["a"].min() == 0 + assert t["a"].null_count() == 1 + + +def test_an_all_null_column_falls_back_to_the_scan(tmp_path): + """Every segment is FLAG_ALL_NULL, so there is nothing left to reduce and + the shortcut must decline rather than answer with the placeholder zeros.""" + t = one_col([None] * 2_000, blosc2.int64(null_storage="mask"), tmp_path / "allnull.b2t") + t.create_index("a", kind="summary") + assert t["a"]._index_summary_minmax("min") is NotImplemented + with pytest.raises(ValueError): + t["a"].min() + + +def test_a_null_written_after_the_build_takes_the_shortcut_away(tmp_path): + """A mask column with no sidecar claims null_aware, so the claim has to stop + holding the moment a null appears. It does, through the ordinary staleness + rule: the write invalidates the index.""" + t = one_col(list(range(5_000)), blosc2.int64(null_storage="mask"), tmp_path / "later.b2t") + t.create_index("a", kind="summary") + assert t["a"]._index_summary_minmax("min") == 0 + + t["a"][0] = None + assert t._get_index_catalog()["a"]["stale"] is True + assert t["a"]._index_summary_minmax("min") is NotImplemented + assert t["a"].min() == 1 + + t.rebuild_index("a") + assert t["a"]._index_summary_minmax("min") == 1 + + +# --------------------------------------------------------------------------- +# where(): OR over a nullable indexed column +# --------------------------------------------------------------------------- + + +def or_table(tmp_path, name, spec, sentinel=None): + """Two int columns whose large values are clustered, so a summary index has + something to prune and the planner prefers it to a scan.""" + n = 2_000_000 + idx = np.arange(n) + a = np.where((idx >= 1000) & (idx < 1100), 5000 + idx, idx % 100).astype(np.int64) + b = np.where((idx >= 1_500_000) & (idx < 1_500_100), 7000 + idx, idx % 100).astype(np.int64) + nulls = (idx % 31) == 0 + + Row = dataclasses.make_dataclass( + "OrRow", [("a", int, blosc2.field(spec)), ("b", int, blosc2.field(blosc2.int64()))] + ) + t = blosc2.CTable(Row, expected_size=n, urlpath=str(tmp_path / name), mode="w") + if sentinel is None: + t.extend({"a": np.ma.MaskedArray(a, mask=nulls), "b": b}) + else: + col = a.copy() + col[nulls] = sentinel + t.extend({"a": col, "b": b}) + t.create_index("a", kind="summary") + t.create_index("b", kind="summary") + expected = int(np.count_nonzero(((a > 4000) & ~nulls) | (b > 6000))) + return t, expected + + +@pytest.mark.parametrize( + ("label", "spec", "sentinel"), + [ + ("mask", blosc2.int64(null_storage="mask"), None), + ("sentinel", blosc2.int64(nullable=True, null_value=-1), -1), + ], +) +def test_indexed_or_over_a_nullable_column_uses_its_index(label, spec, sentinel, tmp_path, monkeypatch): + """It used to bail to a full scan; the segment path evaluates the + (null-aware) predicate, so it is exact without any post-filter.""" + from blosc2.ctable_indexing import _CTableIndexingMixin + + t, expected = or_table(tmp_path, f"or-{label}.b2t", spec, sentinel) + + used = [] + original = _CTableIndexingMixin._try_index_where + + def spy(self, expr): + result = original(self, expr) + used.append(result is not None) + return result + + monkeypatch.setattr(_CTableIndexingMixin, "_try_index_where", spy) + + got = len(t.where("(a > 4000) | (b > 6000)")) + assert got == expected + assert used == [True], "the OR should now be answered from the index" + + +def test_a_row_null_in_one_branch_still_matches_the_other(tmp_path): + """What a global null post-filter would have got wrong, at small scale.""" + rng = np.random.default_rng(6) + n = 2000 + a = [None if i % 31 == 0 else int(rng.integers(0, 1000)) for i in range(n)] + b = [int(rng.integers(0, 1000)) for _ in range(n)] + Row = dataclasses.make_dataclass( + "R", + [ + ("a", int, blosc2.field(blosc2.int64(null_storage="mask"))), + ("b", int, blosc2.field(blosc2.int64())), + ], + ) + t = blosc2.CTable(Row, expected_size=n, urlpath=str(tmp_path / "small.b2t"), mode="w") + t.extend(list(zip(a, b, strict=True))) + t.create_index("a", kind="full") + t.create_index("b", kind="full") + + expected = sum(((av is not None and av > 800) or bv < 200) for av, bv in zip(a, b, strict=True)) + assert len(t.where("(a > 800) | (b < 200)")) == expected + + +def test_a_pipe_inside_a_string_literal_is_not_an_or(): + """The OR test decides whether a nullable column keeps its index, so it + parses rather than searching for a character that string data may contain.""" + from blosc2.ctable_indexing import _expression_has_or + + assert _expression_has_or("(a > 1) | (b < 2)") + assert _expression_has_or("a > 1 or b < 2") + assert not _expression_has_or("name == 'a|b'") + assert not _expression_has_or("(a > 1) & (b < 2)") diff --git a/tests/ctable/test_null_mask_expressions.py b/tests/ctable/test_null_mask_expressions.py index 07b9c9ca6..256d2a24a 100644 --- a/tests/ctable/test_null_mask_expressions.py +++ b/tests/ctable/test_null_mask_expressions.py @@ -278,12 +278,15 @@ def test_mask_bool_filters_directly(): # --------------------------------------------------------------------------- -def test_summary_minmax_shortcut_stays_disabled_for_mask_columns(tmp_path): - """Enabling it would make min() answer differently depending on the index. - - A mask float column's fill is NaN, which the summary builder drops — but so - is a *genuine* NaN, which decision 6 makes a value. The scan therefore - poisons to NaN while the summaries would report a real extremum. +def test_a_genuine_nan_still_keeps_a_mask_float_column_off_the_shortcut(tmp_path): + """Phase 5 disabled the whole column here; Phase 10 narrows that to the NaN. + + Null-aware summaries let a mask float column qualify as a *source* — its + nulls are excluded from the extrema like anything else. What cannot be + shortcut is a genuine NaN, which decision 6 makes a value: the scan poisons + to NaN while the summaries would report a real extremum. ``FLAG_HAS_NAN`` + is raised over the valid rows only, so it now marks exactly that case, and + ``_index_summary_minmax`` declines on it while the source stays usable. """ spec = blosc2.float64(null_storage="mask") Row = dataclasses.make_dataclass("R", [("v", float, blosc2.field(spec))]) @@ -294,13 +297,32 @@ def test_summary_minmax_shortcut_stays_disabled_for_mask_columns(tmp_path): reopened = blosc2.CTable.open(path, mode="a") try: - assert reopened["v"]._summary_minmax_source() is None + assert reopened["v"]._summary_minmax_source() is not None + assert reopened["v"]._index_summary_minmax("min") is NotImplemented # The scanned answer, which the shortcut would have contradicted. assert np.isnan(reopened["v"].min()) finally: reopened.close() +def test_a_nan_free_mask_float_column_now_takes_the_shortcut(tmp_path): + """The other half: with no NaN in the data there is nothing to disagree on.""" + spec = blosc2.float64(null_storage="mask") + Row = dataclasses.make_dataclass("R", [("v", float, blosc2.field(spec))]) + path = str(tmp_path / "nn.b2d") + t = blosc2.CTable(Row, expected_size=5, urlpath=path, mode="w") + t.extend([(1.0,), (2.0,), (5.0,), (None,), (3.0,)]) + t.close() + + reopened = blosc2.CTable.open(path, mode="a") + try: + assert reopened["v"]._index_summary_minmax("min") == 1.0 + assert reopened["v"].min() == 1.0 + assert reopened["v"].max() == 5.0 + finally: + reopened.close() + + def test_sentinel_nan_float_keeps_its_summary_shortcut(tmp_path): """The contrast: there NaN *is* the null, so dropping it is exactly right.""" spec = blosc2.float64(null_value=float("nan")) From 8f608cdaf2b014c94190f3ee4e89bf3b97c3fd1c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 18:29:39 +0200 Subject: [PATCH 12/47] Say plainly what an older reader sees for a version-3 table The previous wording claimed a pre-4.10.2 reader refuses a mask table "with a clear error". It refuses it, but the message is the bare `Unsupported schema version 3`: the hint naming convert_nulls(to='sentinel') was added in the same release that can already read the file, so the reader that actually hits the error never prints it. Also records the two things a reader of this section needs and could not get from it: that version 1 is still what a table with no nullable column stamps, so the compat break is scoped to tables that use the feature; and the process-wide opt-out via NullPolicy, which covers the inferred-schema paths (Arrow, Parquet, CSV) where nobody types a nullability keyword at all. Co-Authored-By: Claude Opus 5 --- RELEASE_NOTES.md | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 51473630a..33c16441a 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -45,9 +45,26 @@ is also `bool_value`'s default, and `NullPolicy(bool_value=255)` carries no information to act on. A bool column that wants a sentinel has to say so with `null_storage` or `column_null_values`. -A table containing a mask column records **schema version 3**; readers older than -4.10.2 refuse it with a clear error rather than misreading it. Pass -`null_storage="sentinel"` for data that has to stay readable by them. +A table containing a mask column records **schema version 3**. Only such tables do: +a table with no nullable column still records version 1, exactly as before, and a +sentinel one does too. Readers older than 4.10.2 refuse a version-3 table rather +than misreading it, but their message is a bare `ValueError: Unsupported schema +version 3` — the hint naming `convert_nulls(to='sentinel')` ships in 4.10.2, so +only readers that can already open the file will print it. + +If some of your data has to stay readable by an earlier release, pin the storage +rather than discovering this downstream. Per column with +`null_storage="sentinel"` (or any explicit `null_value=`), or process-wide, +including for schemas inferred from Arrow, Parquet and CSV: + +```python +with blosc2.null_policy(blosc2.NullPolicy(null_storage="sentinel")): + t = blosc2.CTable.from_parquet("data.parquet") +``` + +That reinstates the sentinel's lossiness — a float column's nulls become `NaN` +again, and a type with no value to spare still cannot be imported — which is the +trade being made. `Column.null_storage` reports where a column keeps its nulls and `info` tags each column (`int64 nullable[mask]`), so `CTable.convert_nulls()` can move columns From 1dbd2a16514d7b4363f14e69e06b73989c1a7791 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 18:37:20 +0200 Subject: [PATCH 13/47] Target 4.11.0, not 4.10.2 The default null storage flips in this release, which is a minor bump rather than a patch one. Retargets every reference: the version string, the release notes heading and its compatibility paragraph, the "since" notes in doc/reference/ctable.rst and CTable's docstrings, the plan's deviation note, and three test docstrings. Also carries the ctable.rst half of the correction made to RELEASE_NOTES.md in the previous commit: an older reader refuses a version-3 table with a bare "Unsupported schema version 3", not with the hint naming convert_nulls, which ships in the same release that can already read the file. Adds the scoping sentence there too -- a table with no nullable column still records version 1. Co-Authored-By: Claude Opus 5 --- RELEASE_NOTES.md | 6 +++--- doc/reference/ctable.rst | 13 ++++++++----- plans/mask-based-nulls.md | 2 +- src/blosc2/ctable.py | 4 ++-- src/blosc2/version.py | 2 +- tests/ctable/test_null_channel.py | 2 +- tests/ctable/test_null_storage_schema.py | 2 +- tests/ctable/test_nullable.py | 2 +- 8 files changed, 18 insertions(+), 15 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 33c16441a..e4052b283 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,6 @@ # Release notes -## Changes from 4.10.1 to 4.10.2 +## Changes from 4.10.1 to 4.11.0 XXX version-specific blurb XXX @@ -47,9 +47,9 @@ information to act on. A bool column that wants a sentinel has to say so with A table containing a mask column records **schema version 3**. Only such tables do: a table with no nullable column still records version 1, exactly as before, and a -sentinel one does too. Readers older than 4.10.2 refuse a version-3 table rather +sentinel one does too. Readers older than 4.11.0 refuse a version-3 table rather than misreading it, but their message is a bare `ValueError: Unsupported schema -version 3` — the hint naming `convert_nulls(to='sentinel')` ships in 4.10.2, so +version 3` — the hint naming `convert_nulls(to='sentinel')` ships in 4.11.0, so only readers that can already open the file will print it. If some of your data has to stay readable by an earlier release, pin the storage diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index 68937e382..6cb4b2c54 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -143,7 +143,7 @@ The two other values are the only representation their kind has: ``"code"`` for a dictionary column, which reserves ``-1``; ``"native"`` for the variable-length container kinds, whose cells simply hold ``None``. -**Mask storage is the default** since 4.10.2, because it is what makes +**Mask storage is the default** since 4.11.0, because it is what makes nullability lossless. A bare ``nullable=True`` — and every nullable column inferred from Arrow, Parquet or CSV — keeps its nulls in a sidecar, so: @@ -157,9 +157,12 @@ inferred from Arrow, Parquet or CSV — keeps its nulls in a sidecar, so: Sentinel storage is supported indefinitely and is one keyword away, per column (``null_storage="sentinel"``, or any explicit ``null_value=``) or globally through :class:`NullPolicy`. It is the right choice when a column has to stay -readable by a Blosc2 release older than 4.10.2: a table containing a mask column -records **schema version 3**, which earlier readers refuse with a clear error -rather than misreading. +readable by a Blosc2 release older than 4.11.0: a table containing a mask column +records **schema version 3**, which earlier readers refuse — with a bare +``ValueError: Unsupported schema version 3``, since the hint naming +:meth:`CTable.convert_nulls` ships in 4.11.0 — rather than misreading. Only a +table that actually contains a mask column is affected; one with no nullable +column still records version 1. Nothing on disk changes. The flip governs *creation* only — opening a stored table never re-resolves anything, so every existing table keeps the storage, @@ -702,7 +705,7 @@ Choosing an index kind back to a full scan to avoid per‑segment evaluation overhead. Indexes on nullable columns - Every index kind stores per‑segment ``min``/``max``, and since 4.10.2 those + Every index kind stores per‑segment ``min``/``max``, and since 4.11.0 those extrema are taken over the rows that carry a **value**: a column's nulls are read from its validity channel — the ``.notnull`` sidecar of a mask column, the reserved value of a sentinel one — and left out. A segment with no diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 7ed41b27c..51ca78ad7 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -68,7 +68,7 @@ answers. the capability ships opt-in first (Phase 6), and the default flips no earlier than one release later (Phase 9), so version-3-capable readers are in circulation before default-created tables require them. *(The two-step staging was not kept — both shipped in - 4.10.2; see the deviation note under §Phasing.)* Sentinel remains fully supported and readable forever, + 4.11.0; see the deviation note under §Phasing.)* Sentinel remains fully supported and readable forever, selectable per column (`null_value=...`, `null_storage="sentinel"`) or globally via `NullPolicy`. Existing on-disk tables keep working unchanged. 2. **V1 scope** = fixed-width scalars + utf8: numeric (incl. **complex**, which gains nullability diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index bc0026a3b..aca6e13a4 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -135,7 +135,7 @@ class NullPolicy: written into the resulting schema, so a stored table stays self-describing and is never re-resolved on open. - Since 4.10.2 the default is a **validity sidecar** (``null_storage="mask"``), + Since 4.11.0 the default is a **validity sidecar** (``null_storage="mask"``), which is what makes nullability lossless. Setting any type-wide sentinel field below asks for in-band storage for the kinds that field covers, so existing ``NullPolicy(float_value=...)`` code keeps its sentinels; passing @@ -192,7 +192,7 @@ class Row: null_storage: Literal["mask", "sentinel"] | None = None #: What an unspecified ``null_storage`` resolves to. ``"mask"`` since - #: 4.10.2: lossless nullability is why the sidecar exists, so it is what a + #: 4.11.0: lossless nullability is why the sidecar exists, so it is what a #: newly created nullable column should get. Sentinel storage stays fully #: supported and is one kwarg away. DEFAULT_NULL_STORAGE: ClassVar[str] = NULL_MASK diff --git a/src/blosc2/version.py b/src/blosc2/version.py index 77f4e2019..5014a7983 100644 --- a/src/blosc2/version.py +++ b/src/blosc2/version.py @@ -1,2 +1,2 @@ -__version__ = "4.10.2.dev0" +__version__ = "4.11.0.dev0" __array_api_version__ = "2024.12" diff --git a/tests/ctable/test_null_channel.py b/tests/ctable/test_null_channel.py index 3838b447c..3d872711a 100644 --- a/tests/ctable/test_null_channel.py +++ b/tests/ctable/test_null_channel.py @@ -78,7 +78,7 @@ def test_kind_of_spec_none(): def test_kind_of_spec_resolves_on_compile(): """``nullable=True`` only picks a channel once the table compiles it. - Bare, it resolves to a mask -- the default since 4.10.2 -- and only an + Bare, it resolves to a mask -- the default since 4.11.0 -- and only an explicit request keeps the sentinel. Either way the spec alone cannot say: it is the resolver that decides. """ diff --git a/tests/ctable/test_null_storage_schema.py b/tests/ctable/test_null_storage_schema.py index 6267b835c..f55ac18b6 100644 --- a/tests/ctable/test_null_storage_schema.py +++ b/tests/ctable/test_null_storage_schema.py @@ -430,7 +430,7 @@ def test_storage_survives_copy(): # --------------------------------------------------------------------------- -# The default flip (4.10.2) +# The default flip (4.11.0) # --------------------------------------------------------------------------- diff --git a/tests/ctable/test_nullable.py b/tests/ctable/test_nullable.py index 1a7db3880..3f328f756 100644 --- a/tests/ctable/test_nullable.py +++ b/tests/ctable/test_nullable.py @@ -109,7 +109,7 @@ def test_null_value_string(): def test_nullable_true_uses_default_null_policy(): """The type-wide sentinels a policy picks, once sentinel storage is asked for. - A bare ``nullable=True`` resolves to a mask since 4.10.2, so this pins the + A bare ``nullable=True`` resolves to a mask since 4.11.0, so this pins the *sentinel* half of the resolution -- which is still what ``null_storage="sentinel"`` and any type-wide policy field select. """ From febc3cd93423fdfc5b1ee4a7a5074df00ffbd94b Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 18:48:56 +0200 Subject: [PATCH 14/47] Give the NaT in the timestamp test a unit NumPy 2.5 deprecated the generic timedelta unit, and a bare np.datetime64("NaT") goes through that path, so under `filterwarnings = error` the construction fails before the library is reached. Ubuntu CI on Python 3.14 caught it; NumPy 2.4.6 does not warn, which is why it passed locally. The warning is raised in the *caller*, not in blosc2: detection is np.isnat, which never cared about the unit. Reproduced against numpy 2.5.1 and pinned by a second test that writes a NaT of each unit, so the unit-independence is asserted rather than assumed. The first test takes its unit from the spec so it tracks the column instead of a literal. Co-Authored-By: Claude Opus 5 --- tests/ctable/test_null_mask_api.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index c2c72b880..1c5c34b0c 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -207,11 +207,22 @@ def test_numpy_float_array_input_has_no_nulls(): def test_nat_reads_as_null_in_a_timestamp_column(): + # The NaT carries a unit: NumPy 2.5 deprecated the *generic* one, so a bare + # np.datetime64("NaT") warns in the caller. Taking the unit from the spec + # keeps this pinned to the column rather than to a literal. spec = blosc2.timestamp(null_storage="mask") - t = simple([np.datetime64("2020-01-01"), np.datetime64("NaT")], spec=spec) + t = simple([np.datetime64("2020-01-01"), np.datetime64("NaT", spec.unit)], spec=spec) assert t["a"].is_null().tolist() == [False, True] +def test_a_nat_of_any_unit_reads_as_null(): + """Detection is np.isnat, so it does not care which unit the caller used.""" + spec = blosc2.timestamp(null_storage="mask") + for unit in ("s", "ms", "us", "ns"): + t = simple([np.datetime64("2020-01-01"), np.datetime64("NaT", unit)], spec=spec) + assert t["a"].is_null().tolist() == [False, True], unit + + # --------------------------------------------------------------------------- # Decision 9 through the public API # --------------------------------------------------------------------------- From d2f586e8d25066756969444d170499e5323510c9 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 18:57:19 +0200 Subject: [PATCH 15/47] Stop re-reading every groupby column once per row test_group_by_agrees_with_sentinel_storage[False-keys2] was the slowest test in CI at 4.46 s. None of it was blosc2: the table is 400 rows, and the helper called as_list(g[name]) *inside* the row loop, so each of the seven result columns was materialized once per output row. as_list is itself O(rows), which made the comparison O(rows^2 * columns) -- and the two-key parametrization, having the most groups, paid the most. Hoisting the column reads out of the loop takes it from 1.08 s to 0.06 s locally (18x), and drops it out of the file's ten slowest tests. The assertions, the fixture and the data are untouched. Also records why or_table() in test_null_aware_indexes.py uses two million rows: the planner only prefers the index over the scan above a threshold measured between 200k and 400k rows, so trimming it would silently cost the "index was used" assertion its margin. It is cheap regardless -- the values are repetitive enough that the table and both indexes come to ~90 KB. Co-Authored-By: Claude Opus 5 --- tests/ctable/test_null_aware_indexes.py | 10 +++++++++- tests/ctable/test_null_mask_sort_groupby.py | 8 ++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/ctable/test_null_aware_indexes.py b/tests/ctable/test_null_aware_indexes.py index 38a873637..7fd96959d 100644 --- a/tests/ctable/test_null_aware_indexes.py +++ b/tests/ctable/test_null_aware_indexes.py @@ -265,7 +265,15 @@ def test_a_null_written_after_the_build_takes_the_shortcut_away(tmp_path): def or_table(tmp_path, name, spec, sentinel=None): """Two int columns whose large values are clustered, so a summary index has - something to prune and the planner prefers it to a scan.""" + something to prune and the planner prefers it to a scan. + + The row count is load-bearing and cannot be trimmed much: the planner's + cost model only prefers the index once the scan it would replace is big + enough, and measured here the switch happens between 200k and 400k rows. + Two million keeps a comfortable margin. It is not expensive despite the + size -- the values are deliberately repetitive, so the whole table plus + both indexes come to about 90 KB on disk. + """ n = 2_000_000 idx = np.arange(n) a = np.where((idx >= 1000) & (idx < 1100), 5000 + idx, idx % 100).astype(np.int64) diff --git a/tests/ctable/test_null_mask_sort_groupby.py b/tests/ctable/test_null_mask_sort_groupby.py index 7604aaad6..cc31947cb 100644 --- a/tests/ctable/test_null_mask_sort_groupby.py +++ b/tests/ctable/test_null_mask_sort_groupby.py @@ -509,12 +509,16 @@ def test_utf8_column_vs_column_comparison_excludes_nulls(): def grouped(t, keys, dropna): """``{key tuple -> {agg -> value}}``, nulls spelled ``None`` throughout.""" g = t.group_by(keys, dropna=dropna, sort=True).agg(**AGGS) + # Each column is read once, not once per row: as_list is O(rows), so + # calling it inside the loop below made this O(rows^2 * columns) and cost + # more than every other test in the file put together. + cols = {name: as_list(g[name]) for name in (*keys, *AGGS)} out = {} for i in range(len(g)): - key = tuple(as_list(g[name])[i] for name in keys) + key = tuple(cols[name][i] for name in keys) row = {} for name in AGGS: - value = as_list(g[name])[i] + value = cols[name][i] if isinstance(value, float) and np.isnan(value): value = None # a non-nullable float output spells "missing" NaN row[name] = value From f7f50793f4053e36dbdee03ae9fc56012fa47b72 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 8 Aug 2026 19:15:37 +0200 Subject: [PATCH 16/47] Let the null-storage suites run on NumPy 1.x blosc2.utf8() raises without numpy.dtypes.StringDType, so on NumPy 1.26 the seven null-storage suites this branch added took ubuntu CI down with 32 collection errors and 4 failures. The distinction that matters: a call inside a test body can be skipped, but a parametrize list is built at *collection* time, so the call raises before any skipif can act and the whole module dies. tests/ctable/utf8_compat.py carries the guard once -- HAVE_UTF8, a needs_utf8 mark, and utf8_spec() returning None where NumPy cannot build a spec. Sibling helper modules already have precedent (tests/b2view/test_basics.py imports tree_store_gen), and the alternative was the same block copied into seven files. Suites that are *entirely* about utf8 keep skipping at module level; this is for the ones where utf8 is one kind among many and the rest must still run. One case needed more than a mark: test_sentinel_storage_is_lossy_where_mask_ storage_is_not asserts that the utf8 sentinel eats a literal "__BLOSC2_NULL__", which is only true when there is a utf8 column. Without StringDType an Arrow string column imports as vlstring, whose nulls are native None, so nothing is lost and the premise does not hold. Verified by running the suite with numpy.dtypes.StringDType deleted, which reproduces the CI failure exactly: 8482 passed, 70 skipped there, and 8766 passed, 29 skipped with it present. Co-Authored-By: Claude Opus 5 --- tests/ctable/test_null_channel.py | 7 ++-- tests/ctable/test_null_mask_api.py | 4 +- tests/ctable/test_null_mask_arrow.py | 14 +++++-- tests/ctable/test_null_mask_sort_groupby.py | 13 ++++++- tests/ctable/test_null_migration.py | 5 ++- tests/ctable/test_null_persistence.py | 2 + tests/ctable/test_null_storage_schema.py | 7 ++-- tests/ctable/utf8_compat.py | 43 +++++++++++++++++++++ 8 files changed, 81 insertions(+), 14 deletions(-) create mode 100644 tests/ctable/utf8_compat.py diff --git a/tests/ctable/test_null_channel.py b/tests/ctable/test_null_channel.py index 3d872711a..aba2a04b2 100644 --- a/tests/ctable/test_null_channel.py +++ b/tests/ctable/test_null_channel.py @@ -20,6 +20,7 @@ import numpy as np import pytest +from utf8_compat import needs_utf8, utf8_spec import blosc2 from blosc2 import CTable @@ -57,8 +58,8 @@ (blosc2.bool(nullable=True, null_value=255), NULL_SENTINEL), (blosc2.bool(), NULL_NONE), # utf8 is a variable-length kind but stores nulls as a sentinel string. - (blosc2.utf8(), NULL_NONE), - (blosc2.utf8(null_value="__NULL__"), NULL_SENTINEL), + pytest.param(utf8_spec(), NULL_NONE, marks=needs_utf8), + pytest.param(utf8_spec(null_value="__NULL__"), NULL_SENTINEL, marks=needs_utf8), # Dictionary and native-None kinds report a channel either way: their # storage can represent a null regardless of the nullable flag. (blosc2.dictionary(), NULL_CODE), @@ -223,7 +224,7 @@ def test_is_null_value(): (blosc2.int64(null_value=-1), int, [1, 2, 3], -1), (blosc2.float64(null_value=float("nan")), float, [1.0, 2.0, 3.0], float("nan")), (blosc2.string(max_length=8, null_value=""), str, ["a", "b", "c"], ""), - (blosc2.utf8(null_value="__NULL__"), str, ["a", "b", "c"], "__NULL__"), + pytest.param(utf8_spec(null_value="__NULL__"), str, ["a", "b", "c"], "__NULL__", marks=needs_utf8), ], ) def test_channel_agrees_with_column_api(spec, annotation, values, null): diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index 1c5c34b0c..3b8d435d9 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -28,6 +28,7 @@ import numpy as np import pytest +from utf8_compat import needs_utf8, utf8_spec import blosc2 @@ -76,7 +77,7 @@ def simple(values, spec=None, capacity=32): ("bool", blosc2.bool(null_storage="mask"), True), ("string", blosc2.string(max_length=4, null_storage="mask"), "abcd"), ("bytes", blosc2.bytes(max_length=4, null_storage="mask"), b"abcd"), - ("utf8", blosc2.utf8(null_storage="mask"), "hello"), + pytest.param("utf8", utf8_spec(null_storage="mask"), "hello", marks=needs_utf8), ] @@ -131,6 +132,7 @@ def test_string_keeps_its_declared_width(): assert t["a"].dtype == np.dtype("U4") +@needs_utf8 def test_utf8_accepts_text_no_sentinel_could_survive(): tricky = ["", "\x00", "__BLOSC2_NULL__", "🎉x"] t = simple([*tricky, None], spec=blosc2.utf8(null_storage="mask")) diff --git a/tests/ctable/test_null_mask_arrow.py b/tests/ctable/test_null_mask_arrow.py index e3cb6c55c..8bd22b7cc 100644 --- a/tests/ctable/test_null_mask_arrow.py +++ b/tests/ctable/test_null_mask_arrow.py @@ -29,6 +29,7 @@ import numpy as np import pytest +from utf8_compat import needs_utf8, utf8_spec import blosc2 @@ -168,11 +169,15 @@ def test_ndarray_column_round_trip(): [ # -128 is the sentinel int8 picks, so real -128 data reads back as null. ("int8_min", pa.array([-128, None, 127], type=pa.int8()), [None, None, 127]), - # "__BLOSC2_NULL__" is literally the utf8 sentinel. - ( + # "__BLOSC2_NULL__" is literally the utf8 sentinel. Only on NumPy >= 2: + # without StringDType an Arrow string column imports as vlstring, whose + # nulls are native None, so there is no sentinel to collide with and + # nothing is lost. + pytest.param( "utf8_sentinel_literal", pa.array(["", "__BLOSC2_NULL__", None], type=pa.string()), ["", None, None], + marks=needs_utf8, ), ], ) @@ -375,11 +380,12 @@ def build(spec, values, capacity=32): [b"abcd", None, b""], [b"abcd", None, b""], ), - ( + pytest.param( "utf8", - blosc2.utf8(null_storage="mask"), + utf8_spec(null_storage="mask"), ["", "\x00", None], ["", "\x00", None], + marks=needs_utf8, ), ] diff --git a/tests/ctable/test_null_mask_sort_groupby.py b/tests/ctable/test_null_mask_sort_groupby.py index cc31947cb..88a135be1 100644 --- a/tests/ctable/test_null_mask_sort_groupby.py +++ b/tests/ctable/test_null_mask_sort_groupby.py @@ -33,6 +33,7 @@ import numpy as np import pytest +from utf8_compat import needs_utf8, utf8_spec import blosc2 @@ -64,7 +65,7 @@ def one_col(values, spec, capacity=64, urlpath=None): ("int64", blosc2.int64, {}, -(2**62)), ("float64", blosc2.float64, {}, np.nan), ("string", blosc2.string, {"max_length": 4}, "\x7f\x7f"), - ("utf8", blosc2.utf8, {}, "__BLOSC2_NULL__"), + pytest.param("utf8", blosc2.utf8, {}, "__BLOSC2_NULL__", marks=needs_utf8), ] @@ -280,6 +281,7 @@ def test_sorted_slice_keeps_its_window_read_when_there_are_no_nulls(tmp_path): # --------------------------------------------------------------------------- +@needs_utf8 def test_utf8_rank_index_separates_nulls_from_genuine_empty_strings(tmp_path): """The landmine this phase was warned about, and it is a real one. @@ -297,6 +299,7 @@ def test_utf8_rank_index_separates_nulls_from_genuine_empty_strings(tmp_path): assert as_list(t.sort_by("a")["a"]) == ["", "", "a", "b", None, None] +@needs_utf8 def test_utf8_rank_arrays_stamps_nulls_with_the_null_rank(): """Directly, since this is where the recoding happens.""" from blosc2.ctable_indexing import _utf8_rank_arrays @@ -311,6 +314,7 @@ def test_utf8_rank_arrays_stamps_nulls_with_the_null_rank(): assert int(ranks[2]) == 0 # the genuine "" keeps rank 0 +@needs_utf8 def test_a_pre_mask_utf8_rank_index_is_treated_as_stale(tmp_path): """An index built before nulls got their own rank cannot be trusted. @@ -327,6 +331,7 @@ def test_a_pre_mask_utf8_rank_index_is_treated_as_stale(tmp_path): assert t._utf8_rank_index_stale("a", meta) is True +@needs_utf8 def test_a_sentinel_utf8_rank_index_stays_fresh_without_the_flag(tmp_path): """The staleness rule must not fire for sentinel columns. @@ -467,12 +472,14 @@ def test_a_guard_is_only_emitted_where_the_fill_could_match(): "query", ["a < 'b'", "a > 'b'", "a != 'a'", "a == ''", "startswith(a, 'a')"], ) +@needs_utf8 def test_utf8_predicates_agree_between_storages(query): values = ["e", None, "a", "z", "", "b"] mask, sent = pair(values, blosc2.utf8, {}, "__BLOSC2_NULL__") assert as_list(mask.where(query)["a"]) == as_list(sent.where(query)["a"]) +@needs_utf8 def test_utf8_span_driver_excludes_mask_nulls(): """The span driver materializes nulls to ``""`` and re-applies nullity. @@ -484,6 +491,7 @@ def test_utf8_span_driver_excludes_mask_nulls(): assert as_list(t.where("startswith(a, '')")["a"]) == ["ax", "bx"] +@needs_utf8 def test_utf8_column_vs_column_comparison_excludes_nulls(): t = table( [("a", "a"), (None, "a"), ("b", None), ("c", "c")], @@ -651,6 +659,7 @@ def test_group_by_a_mask_float_key_groups_nan_with_the_nulls(): assert as_list(g["total"]) == [6, 1] +@needs_utf8 def test_group_by_utf8_mask_key_separates_nulls_from_empty_strings(): t = table( [("", 1), (None, 2), ("", 4)], @@ -669,7 +678,7 @@ def test_group_by_utf8_mask_key_separates_nulls_from_empty_strings(): (blosc2.uint8(null_storage="mask"), [255, None, 0, 255]), (blosc2.bool(null_storage="mask"), [True, None, False, True]), (blosc2.bytes(max_length=2, null_storage="mask"), [b"aa", None, b"", b"aa"]), - (blosc2.utf8(null_storage="mask"), ["aa", None, "", "aa"]), + pytest.param(utf8_spec(null_storage="mask"), ["aa", None, "", "aa"], marks=needs_utf8), ], ) def test_group_by_a_mask_key_of_every_v1_kind(spec, values): diff --git a/tests/ctable/test_null_migration.py b/tests/ctable/test_null_migration.py index 7f0d7d549..cef2612af 100644 --- a/tests/ctable/test_null_migration.py +++ b/tests/ctable/test_null_migration.py @@ -25,6 +25,7 @@ import numpy as np import pytest +from utf8_compat import needs_utf8 import blosc2 @@ -69,7 +70,7 @@ def _scalar(value): ("bool", blosc2.bool, {}, [True, None, False]), ("string", blosc2.string, {"max_length": 4}, ["ab", None, "cd"]), ("bytes", blosc2.bytes, {"max_length": 4}, [b"ab", None, b"cd"]), - ("utf8", blosc2.utf8, {}, ["ab", None, "cdefgh"]), + pytest.param("utf8", blosc2.utf8, {}, ["ab", None, "cdefgh"], marks=needs_utf8), ( "timestamp", blosc2.timestamp, @@ -193,12 +194,14 @@ def test_to_sentinel_refuses_a_full_range_int8(): t.convert_nulls("a", to="sentinel") +@needs_utf8 def test_to_sentinel_refuses_utf8_holding_the_sentinel_string(): t = one_col(["__BLOSC2_NULL__", None], blosc2.utf8(null_storage="mask")) with pytest.raises(ValueError, match="already contains"): t.convert_nulls("a", to="sentinel") +@needs_utf8 def test_to_sentinel_accepts_a_different_sentinel_instead(): """The refusal names the offending value, and a free one is accepted.""" t = one_col(["__BLOSC2_NULL__", None], blosc2.utf8(null_storage="mask")) diff --git a/tests/ctable/test_null_persistence.py b/tests/ctable/test_null_persistence.py index 82985a432..d6715e6bd 100644 --- a/tests/ctable/test_null_persistence.py +++ b/tests/ctable/test_null_persistence.py @@ -29,6 +29,7 @@ import numpy as np import pytest +from utf8_compat import needs_utf8 import blosc2 from blosc2.ctable_storage import _NOTNULL_SUFFIX, FileTableStorage @@ -171,6 +172,7 @@ def test_sidecar_grid_survives_a_chunk_override(tmp_path): copied.close() +@needs_utf8 def test_utf8_sidecar_falls_back_to_the_table_grid(): """utf8 offsets carry ``n + 1`` entries, so they are not the grid to pin to.""" Row = dataclasses.make_dataclass("Utf8Row", [("u", str, blosc2.field(blosc2.utf8(null_storage="mask")))]) diff --git a/tests/ctable/test_null_storage_schema.py b/tests/ctable/test_null_storage_schema.py index f55ac18b6..10bf1287d 100644 --- a/tests/ctable/test_null_storage_schema.py +++ b/tests/ctable/test_null_storage_schema.py @@ -21,6 +21,7 @@ import numpy as np import pytest +from utf8_compat import HAVE_UTF8, needs_utf8, utf8_spec import blosc2 from blosc2 import CTable @@ -40,7 +41,7 @@ ("timestamp", blosc2.timestamp, {}), ("string", blosc2.string, {"max_length": 4}), ("bytes", blosc2.bytes, {"max_length": 4}), - ("utf8", blosc2.utf8, {}), + pytest.param("utf8", blosc2.utf8, {}, marks=needs_utf8), ] @@ -350,7 +351,7 @@ def test_sentinel_storage_is_unchanged_when_asked_for(): (blosc2.bool(), False), (blosc2.string(max_length=4), ""), (blosc2.bytes(max_length=4), b""), - (blosc2.utf8(), ""), + pytest.param(utf8_spec(), "", marks=needs_utf8), ], ) def test_fill_value_for(spec, expected): @@ -442,7 +443,7 @@ def test_a_bare_nullable_column_gets_a_mask(): (blosc2.bool(nullable=True), bool), (blosc2.string(max_length=4, nullable=True), str), (blosc2.bytes(max_length=4, nullable=True), bytes), - (blosc2.utf8(nullable=True), str), + *([(blosc2.utf8(nullable=True), str)] if HAVE_UTF8 else []), (blosc2.timestamp(nullable=True), object), ]: col = _resolved(spec, annotation) diff --git a/tests/ctable/utf8_compat.py b/tests/ctable/utf8_compat.py new file mode 100644 index 000000000..41afc0698 --- /dev/null +++ b/tests/ctable/utf8_compat.py @@ -0,0 +1,43 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Letting utf8 cases sit in suites that must still run on NumPy 1.x. + +``blosc2.utf8()`` raises on NumPy < 2.0, which has no ``StringDType``. That is +fine inside a test body -- :data:`needs_utf8` skips it -- but a parametrize list +is built at *collection* time, so the call raises before any ``skipif`` can act +and takes the whole module down with it. :func:`utf8_spec` returns ``None`` +there instead, and the mark keeps the placeholder from ever being dereferenced. + +Suites that are entirely about utf8 do not need this: they skip at module level +(see ``test_utf8.py``). This is for the null-storage suites, where utf8 is one +kind among many and everything else must still be exercised. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +import blosc2 + +#: Whether this NumPy can back a utf8 column at all. +HAVE_UTF8 = hasattr(np.dtypes, "StringDType") + +#: Skip a test that builds a utf8 column in its body. +needs_utf8 = pytest.mark.skipif( + not HAVE_UTF8, reason="utf8 columns require NumPy >= 2.0 (numpy.dtypes.StringDType)" +) + + +def utf8_spec(**kwargs): + """``blosc2.utf8(**kwargs)``, or ``None`` where NumPy cannot build one. + + Pair the ``None`` with ``marks=needs_utf8`` on the parametrize entry, so the + placeholder is collected but never used. + """ + return blosc2.utf8(**kwargs) if HAVE_UTF8 else None From 6c04dca4b4b4136f67f59084ae34cac6b6fa489f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 09:30:43 +0200 Subject: [PATCH 17/47] Keep an empty text cell apart from a missing one in CSV An empty CSV field cannot mean both "" and missing, so a mask-backed text column wrote a valid "" exactly like a null, and from_csv read both back as null. Such a column now writes \N for a null and \E for a valid "", escaping any value already shaped like one. Reading text also stops calling strip() before testing for an empty field, so whitespace-only text stays a value. Co-Authored-By: Claude Opus 5 --- src/blosc2/ctable.py | 77 +++++++++++++++++++++++++------- tests/ctable/test_csv_interop.py | 16 +++++++ 2 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index aca6e13a4..794e10a16 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -4308,6 +4308,34 @@ def col(name: str) -> ColExpr: return ColExpr(lambda t: t[name], name) +# CSV text escapes: an empty field is CSV for "missing", which leaves a valid "" +# in a mask-backed text column with nothing to write. \N marks a null and \E a +# valid "", and a value already shaped like one grows a backslash (\N -> \\N). +_CSV_TEXT_TOKEN = re.compile(r"^\\+[NE]$") + + +def _csv_text_escape(value, is_null: bool) -> str: + """Encode one text cell (and its validity) as an unambiguous CSV field.""" + if is_null: + return "\\N" + if isinstance(value, bytes): + value = value.decode() + else: + value = str(value) + if value == "": + return "\\E" + return "\\" + value if _CSV_TEXT_TOKEN.match(value) else value + + +def _csv_text_unescape(field: str) -> str | None: + """Inverse of :func:`_csv_text_escape`; ``None`` means the cell is missing.""" + if field in ("", "\\N"): + return None + if field == "\\E": + return "" + return field[1:] if _CSV_TEXT_TOKEN.match(field) else field + + class CTable(_CTableIndexingMixin, Generic[RowT]): """Columnar compressed table with typed columns and row-oriented access.""" @@ -9290,6 +9318,12 @@ def to_csv(self, path: str | None = None, *, header: bool = True, sep: str = "," Fixed-shape ndarray column cells are serialised as JSON arrays for readability and shape safety (e.g. ``"[1.0, 2.0, 3.0]"``). + Nulls are written as empty fields, except in a mask-backed **text** + column, where an empty field cannot mean both ``""`` and missing: there + a null is written as ``\\N`` and a valid ``""`` as ``\\E`` (a value that + already looks like either grows a backslash). :meth:`from_csv` reads + both back, so ``""`` and ``None`` survive a round trip. + Parameters ---------- path: @@ -9325,6 +9359,13 @@ def to_csv(self, path: str | None = None, *, header: bool = True, sep: str = "," else: json_strings.append(json.dumps(arr[i].tolist())) arrays.append(json_strings) + elif col.null_storage == NULL_MASK and col.dtype.kind in ("U", "S"): + # An empty field cannot mean both "" and missing, so a mask-backed + # text column writes \N for a null and \E for a valid "", escaping + # any value that would collide (\N -> \\N). from_csv undoes both. + arrays.append( + [_csv_text_escape(v, is_null) for v, is_null in zip(col[:], col.is_null(), strict=True)] + ) elif col.null_storage == NULL_MASK and col.null_count(): # An empty field is CSV for missing, and for a mask column it is # the *only* way to say it -- writing the fill would come back as @@ -9403,29 +9444,33 @@ def _csv_col_to_array(raw: list[str], col, nv) -> tuple[np.ndarray, np.ndarray | and raise. *valid* is ``None`` when nothing was missing, or when the column is not nullable at all. """ - missing = np.array([v.strip() == "" for v in raw], dtype=np.bool_) + uses_mask = getattr(col.spec, "uses_mask", False) + if col.dtype.kind in ("U", "S"): + # Text: only an exactly empty field is missing (whitespace-only text + # is a real value), plus the \N / \E escapes to_csv writes for a mask + # column, where "" and missing would otherwise look the same. + decoded = [_csv_text_unescape(v) if uses_mask else (None if v == "" else v) for v in raw] + missing = np.array([v is None for v in decoded], dtype=np.bool_) + raw = ["" if v is None else v for v in decoded] + else: + missing = np.array([v.strip() == "" for v in raw], dtype=np.bool_) placeholder = nv valid = None - if nv is None and getattr(col.spec, "uses_mask", False) and missing.any(): + if nv is None and uses_mask and missing.any(): placeholder = fill_value_for(col.spec) valid = ~missing - if col.dtype == np.bool_: - - def _parse(v, _fill=placeholder): - stripped = v.strip() - if stripped == "" and _fill is not None: - return _fill - return stripped in ("True", "true", "1") + def _fill_missing(values): + if placeholder is None: + return values + return [placeholder if m else v for v, m in zip(values, missing, strict=True)] - return np.array([_parse(v) for v in raw], dtype=np.bool_), valid + if col.dtype == np.bool_: + parsed = [v.strip() in ("True", "true", "1") for v in raw] + return np.array(_fill_missing(parsed), dtype=np.bool_), valid if col.dtype.kind == "S": - prepared: list = [ - placeholder if (v.strip() == "" and placeholder is not None) else v.encode() for v in raw - ] - return np.array(prepared, dtype=col.dtype), valid - prepared2 = [placeholder if (v.strip() == "" and placeholder is not None) else v for v in raw] - return np.array(prepared2, dtype=col.dtype), valid + return np.array(_fill_missing([v.encode() for v in raw]), dtype=col.dtype), valid + return np.array(_fill_missing(raw), dtype=col.dtype), valid @classmethod def from_csv( diff --git a/tests/ctable/test_csv_interop.py b/tests/ctable/test_csv_interop.py index 8a904d807..650ce83bc 100644 --- a/tests/ctable/test_csv_interop.py +++ b/tests/ctable/test_csv_interop.py @@ -538,5 +538,21 @@ def test_from_pandas_multi_dim_ndarray_roundtrip(): assert t2["label"][:].tolist() == t["label"][:].tolist() +@dataclass +class MaskTextRow: + text: str = blosc2.field(blosc2.string(max_length=16, nullable=True, null_storage="mask"), default="") + + +def test_csv_mask_text_roundtrip_empty_vs_null(tmp_csv): + """A mask-backed text column keeps "" apart from a null (and from whitespace).""" + values = ["", " ", None, "ok", "\\N", "\\E", "\\\\N", "N"] + t = CTable(MaskTextRow, new_data=[(v,) for v in values]) + t.to_csv(tmp_csv) + t2 = CTable.from_csv(tmp_csv, MaskTextRow) + + assert [None if n else v for v, n in zip(t2["text"][:], t2["text"].is_null(), strict=True)] == values + assert t2["text"].null_count() == 1 + + if __name__ == "__main__": pytest.main(["-v", __file__]) From 425061317ab3aa27b11277dba43e3a2f185ef8ee Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 10:20:46 +0200 Subject: [PATCH 18/47] Stop losing text and nulls on the way through CSV and extend() Two silent-data-loss paths, both where nullity or row identity travels beside the values rather than in them. **CSV and utf8.** Both halves of the CSV path keyed off a fixed element dtype, which utf8 does not have: a utf8 Column reports StringDType() (kind "T", not the "U"/"S" a fixed-width text column reports), and its CompiledColumn reports None. So from_csv raised AttributeError on any utf8 column, nullable or not, and to_csv skipped the \N / \E escapes and wrote a valid "" exactly like a null. to_csv now gates on _csv_is_text_column; from_csv creates utf8 through create_varlen_scalar_column and fills it via _csv_text_col_to_cells, with the shared text decoding factored into _csv_decode_text. A kind CSV genuinely cannot represent now raises naming itself, instead of failing on None deep inside the conversion. A sentinel text column still reads an empty field back as its sentinel, so a genuine "" becomes null. That is long-standing string/bytes behaviour which utf8 now matches, and changing it would alter how blank fields in third-party CSVs are read; left alone deliberately. **extend() from a CTable.** The raw source._cols[name][:n_rows] slice got two things wrong. It dropped validity: under mask storage nullity lives in a sidecar, so every null arrived as its fill -- a plausible 0 or "" rather than an error -- where the sentinel and list-of-rows forms of the same rows both kept it. And it read physical slots by a logical count, so a source with deleted rows, or a sorted view, copied the wrong rows entirely (storage-independent, and wrong on main too). _batch_columns_from_table now hands validity over separately and gathers at live positions, keeping the plain slice for the dense case behind two O(1) checks. tests/ctable: 2189 passed, 1 xfailed. Co-Authored-By: Claude Opus 5 --- src/blosc2/ctable.py | 154 ++++++++++++++++++++++++++--- tests/ctable/test_csv_interop.py | 50 ++++++++++ tests/ctable/test_null_mask_api.py | 81 +++++++++++++++ 3 files changed, 270 insertions(+), 15 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 794e10a16..8daecbb33 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -4336,6 +4336,18 @@ def _csv_text_unescape(field: str) -> str | None: return field[1:] if _CSV_TEXT_TOKEN.match(field) else field +def _csv_is_text_column(col) -> bool: + """True when *col* holds text, so an empty CSV field would be ambiguous. + + Takes a :class:`Column`, and must ask ``is_utf8`` rather than only reading + the dtype: a utf8 column reports ``StringDType()``, whose kind is ``"T"`` + and not the ``"U"``/``"S"`` a fixed-width text column reports. Testing the + kind alone left utf8 -- the one text kind with no safe sentinel, and so the + kind that most needs the escapes -- writing ``""`` and a null identically. + """ + return col.is_utf8 or col.dtype.kind in ("U", "S") + + class CTable(_CTableIndexingMixin, Generic[RowT]): """Columnar compressed table with typed columns and row-oriented access.""" @@ -9359,7 +9371,7 @@ def to_csv(self, path: str | None = None, *, header: bool = True, sep: str = "," else: json_strings.append(json.dumps(arr[i].tolist())) arrays.append(json_strings) - elif col.null_storage == NULL_MASK and col.dtype.kind in ("U", "S"): + elif col.null_storage == NULL_MASK and _csv_is_text_column(col): # An empty field cannot mean both "" and missing, so a mask-backed # text column writes \N for a null and \E for a valid "", escaping # any value that would collide (\N -> \\N). from_csv undoes both. @@ -9446,12 +9458,7 @@ def _csv_col_to_array(raw: list[str], col, nv) -> tuple[np.ndarray, np.ndarray | """ uses_mask = getattr(col.spec, "uses_mask", False) if col.dtype.kind in ("U", "S"): - # Text: only an exactly empty field is missing (whitespace-only text - # is a real value), plus the \N / \E escapes to_csv writes for a mask - # column, where "" and missing would otherwise look the same. - decoded = [_csv_text_unescape(v) if uses_mask else (None if v == "" else v) for v in raw] - missing = np.array([v is None for v in decoded], dtype=np.bool_) - raw = ["" if v is None else v for v in decoded] + raw, missing = CTable._csv_decode_text(raw, uses_mask) else: missing = np.array([v.strip() == "" for v in raw], dtype=np.bool_) placeholder = nv @@ -9472,6 +9479,43 @@ def _fill_missing(values): return np.array(_fill_missing([v.encode() for v in raw]), dtype=col.dtype), valid return np.array(_fill_missing(raw), dtype=col.dtype), valid + @staticmethod + def _csv_decode_text(raw: list[str], uses_mask: bool) -> tuple[list[str], np.ndarray]: + """Split raw text fields into ``(texts, missing)``. + + Only an *exactly* empty field is missing -- whitespace-only text stays a + real value -- plus the ``\\N``/``\\E`` escapes :meth:`to_csv` writes for a + mask column, where ``""`` and missing would otherwise look the same. + + *texts* carries ``""`` wherever *missing* is True; what a null slot ends + up holding is the caller's decision, since it differs per storage. + """ + decoded = [_csv_text_unescape(v) if uses_mask else (None if v == "" else v) for v in raw] + missing = np.array([v is None for v in decoded], dtype=np.bool_) + return ["" if v is None else v for v in decoded], missing + + @staticmethod + def _csv_text_col_to_cells(raw: list[str], col, nv) -> tuple[list[str], np.ndarray | None]: + """Convert raw CSV fields to ``(cells, valid)`` for a **utf8** column. + + The same missing/fill/validity split as :meth:`_csv_col_to_array`, but + utf8 is fed a list of Python strings instead of a typed array: it has no + fixed element dtype to cast through, which is exactly why it cannot go + through that method at all. + """ + uses_mask = getattr(col.spec, "uses_mask", False) + texts, missing = CTable._csv_decode_text(raw, uses_mask) + if not missing.any(): + return texts, None + if nv is not None: + # Sentinel storage: the null travels in band, as it does on export. + return [nv if m else v for v, m in zip(texts, missing, strict=True)], None + if not uses_mask: + # Not nullable at all, so an empty field is a genuine empty string. + return texts, None + fill = fill_value_for(col.spec) + return [fill if m else v for v, m in zip(texts, missing, strict=True)], ~missing + @classmethod def from_csv( cls, @@ -9488,6 +9532,10 @@ def from_csv( each column is bulk-written into a pre-allocated NDArray (one slice assignment per column, no ``extend()``). + Supported column kinds are the fixed-width scalars, :func:`~blosc2.utf8`, + and fixed-shape :func:`~blosc2.ndarray`; anything else raises rather than + being read as something it is not. + Parameters ---------- path: @@ -9511,7 +9559,8 @@ def from_csv( TypeError If *row_cls* is not a dataclass. ValueError - If a row has a different number of fields than the schema. + If a row has a different number of fields than the schema, or if + *row_cls* declares a column kind CSV cannot represent. """ import csv @@ -9547,6 +9596,23 @@ def from_csv( ) new_cols: dict[str, blosc2.NDArray] = {} for col in schema.columns: + if cls._is_utf8_column(col): + # utf8 has no fixed element dtype, so it cannot be created (or + # written) as a plain NDArray the way every other CSV-supported + # kind is. Its own storage handles both. + new_cols[col.name] = mem_storage.create_varlen_scalar_column( + col.name, spec=col.spec, cparams=None, dparams=None + ) + continue + if col.dtype is None: + # Everything below indexes off a fixed element dtype; a kind + # without one would otherwise fail deep in the conversion with + # an AttributeError on None. + raise ValueError( + f"Column {col.name!r}: from_csv() does not support " + f"{type(col.spec).__name__} columns. Supported kinds are the " + f"fixed-width scalars, utf8, and fixed-shape ndarray." + ) shape = cls._column_physical_shape(col, capacity) if col.name in aligned_names: chunks, blocks = shared_chunks, shared_blocks @@ -9586,12 +9652,19 @@ def from_csv( if n > 0: for i, col in enumerate(schema.columns): + nv = getattr(col.spec, "null_value", None) if isinstance(col.spec, NDArraySpec): arr, valid = cls._csv_ndarray_col_to_array(col_data[i], col) + new_cols[col.name][:n] = arr + elif cls._is_utf8_column(col): + cells, valid = cls._csv_text_col_to_cells(col_data[i], col, nv) + # Appended, not slice-assigned: a utf8 column grows to its + # logical length rather than being pre-sized to capacity. + new_cols[col.name].extend(cells) + new_cols[col.name].flush() else: - nv = getattr(col.spec, "null_value", None) arr, valid = cls._csv_col_to_array(col_data[i], col, nv) - new_cols[col.name][:n] = arr + new_cols[col.name][:n] = arr if valid is not None: # A mask column's empty cells: the values above hold the # fill, and this is the only record that they were missing. @@ -14021,6 +14094,49 @@ def delete(self, ind: int | slice | str | Iterable) -> None: self._last_pos = None # last live row deleted; recalculate on next write self._storage.bump_visibility_epoch() + def _batch_columns_from_table(self, source: CTable, names) -> tuple[dict, dict]: + """Read *source*'s live rows as ``(raw_columns, source_valid)`` for :meth:`extend`. + + Two things a plain ``source._cols[name][:n_rows]`` slice gets wrong, both + of which turn into silently altered data rather than an error: + + * **The rows.** A source with deleted rows -- or a sorted/filtered view -- + keeps its live rows scattered through the physical extent, so the + leading ``n_rows`` physical slots are not the rows being copied. The + dense case still takes the plain slice, which is the common one and + costs no position scan. + * **The nulls.** Under mask storage nullity lives beside the values, not + in them, so a raw slice copies a null's *fill* -- an ordinary-looking + ``0`` / ``""`` / ``NaT`` -- and drops the fact that it was missing. + Sentinel and native-``None`` sources need nothing here: their nulls + travel with the values. + + *source_valid* maps a column to its validity only where something was + actually null, matching what ``NullChannel.coerce_batch`` returns. + """ + # Dense means every live row sits at its own index below the write + # watermark, so the slice and the gather agree. Both terms are O(1). + dense = source.base is None and source._resolve_last_pos() == source._n_rows + positions = None if dense else source._live_positions_from_valid_rows_chunks() + + raw_columns: dict = {} + source_valid: dict = {} + for name in names: + if name not in source._cols: + continue + arr = source._cols[name] + raw_columns[name] = arr[: source._n_rows] if dense else arr[positions] + spec = source._schema.columns_by_name[name].spec + if not getattr(spec, "uses_mask", False): + continue + mask = source._null_mask(name) + if mask is None: + continue # no sidecar means the column has never held a null + valid = np.asarray(mask[: source._n_rows] if dense else mask[positions]) + if not valid.all(): + source_valid[name] = valid + return raw_columns, source_valid + def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> None: # noqa: C901 """Append multiple rows at once. @@ -14069,14 +14185,15 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> input_col_names = self._append_input_col_names new_nrows = 0 provided_names: set[str] = set() + # Validity carried over from a CTable source, whose nullity does not + # travel with its raw values under mask storage. Empty for every other + # input shape, which spells its nulls in band or as ``None`` cells. + source_valid: dict[str, np.ndarray] = {} if hasattr(data, "_cols") and hasattr(data, "_n_rows"): new_nrows = data._n_rows - raw_columns = {} - for name in current_col_names: - if name in data._cols: - raw_columns[name] = data._cols[name][: data._n_rows] - provided_names.add(name) + raw_columns, source_valid = self._batch_columns_from_table(data, current_col_names) + provided_names = set(raw_columns) else: if isinstance(data, dict): if any(isinstance(v, dict) for v in data.values()): @@ -14205,6 +14322,13 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> else: scalar_processed_cols[name] = np.ascontiguousarray(raw, dtype=target_dtype) + # A CTable source hands over its validity separately, because its raw + # values carry none: coerce_batch sees a fully typed array and rightly + # reports nothing null. setdefault, not update, so an input that *did* + # express its own nulls keeps them. + for name, valid in source_valid.items(): + batch_valid.setdefault(name, valid) + end_pos = start_pos + new_nrows if self.auto_compact and end_pos >= len(self._valid_rows): diff --git a/tests/ctable/test_csv_interop.py b/tests/ctable/test_csv_interop.py index 650ce83bc..d02097211 100644 --- a/tests/ctable/test_csv_interop.py +++ b/tests/ctable/test_csv_interop.py @@ -8,11 +8,13 @@ """Tests for CTable.to_csv() and CTable.from_csv().""" import csv +import dataclasses import os from dataclasses import dataclass import numpy as np import pytest +from utf8_compat import needs_utf8, utf8_spec import blosc2 from blosc2 import CTable @@ -554,5 +556,53 @@ def test_csv_mask_text_roundtrip_empty_vs_null(tmp_csv): assert t2["text"].null_count() == 1 +# --------------------------------------------------------------------------- +# utf8 columns +# --------------------------------------------------------------------------- +# +# utf8 reports StringDType(), whose kind is "T" -- not the "U"/"S" a fixed-width +# text column reports, and not a fixed element dtype at all on the schema side. +# Both halves of the CSV path used to key off that dtype, so utf8 was the one +# text kind that neither escaped its empty cells nor could be read back. + + +def utf8_row(**kwargs): + """A one-utf8-column dataclass. Only ever called from a ``needs_utf8`` test.""" + return dataclasses.make_dataclass("Utf8Row", [("text", str, blosc2.field(utf8_spec(**kwargs)))]) + + +@needs_utf8 +def test_csv_utf8_column_roundtrips(tmp_csv): + """A plain utf8 column could not be read back at all -- from_csv raised.""" + row_cls = utf8_row() + values = ["x", "", " ", "unicode: \u00e9\u4e2d"] + t = CTable(row_cls, new_data=[(v,) for v in values]) + t.to_csv(tmp_csv) + t2 = CTable.from_csv(tmp_csv, row_cls) + assert list(t2["text"][:]) == values + + +@needs_utf8 +def test_csv_mask_utf8_keeps_empty_apart_from_null(tmp_csv): + """Same contract as the fixed-width text column above, for utf8.""" + row_cls = utf8_row(null_storage="mask") + values = ["", " ", None, "ok", "\\N", "\\E", "\\\\N", "N"] + t = CTable(row_cls, new_data=[(v,) for v in values]) + t.to_csv(tmp_csv) + t2 = CTable.from_csv(tmp_csv, row_cls) + + assert [None if n else v for v, n in zip(t2["text"][:], t2["text"].is_null(), strict=True)] == values + assert t2["text"].null_count() == 1 + + +def test_csv_rejects_a_kind_it_cannot_read(tmp_csv): + """A clear refusal, not an AttributeError from deep inside the conversion.""" + row_cls = dataclasses.make_dataclass("VlRow", [("v", str, blosc2.field(blosc2.vlstring()))]) + with open(tmp_csv, "w") as f: + f.write("v\nq\n") + with pytest.raises(ValueError, match="does not support"): + CTable.from_csv(tmp_csv, row_cls) + + if __name__ == "__main__": pytest.main(["-v", __file__]) diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index 3b8d435d9..ea3a1aa0b 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -518,3 +518,84 @@ def build(): assert ref() is None finally: gc.enable() + + +# --------------------------------------------------------------------------- +# Extending from another CTable +# --------------------------------------------------------------------------- +# +# A CTable source is the one input shape whose nulls do not travel with its +# values: under mask storage nullity lives in a sidecar, so copying the raw +# column alone turns every null into its fill -- a plausible-looking 0 or "", +# not an error. The sentinel and list-of-rows forms below are the controls +# that say what the answer has to be. + + +def test_extend_from_a_table_carries_the_nulls_over(): + src = simple([1, None, 3]) + dst = simple([]) + dst.extend(src) + assert dst["a"][:].tolist() == [1, 0, 3] + assert dst["a"].is_null().tolist() == [False, True, False] + assert dst["a"].null_count() == 1 + + +def test_extend_from_a_table_agrees_with_the_other_input_shapes(): + """The same rows, spelled three ways, must land identically.""" + from_table = simple([]) + from_table.extend(simple([1, None, 3])) + + from_rows = simple([]) + from_rows.extend([(1,), (None,), (3,)]) + + sentinel = table([], a=blosc2.int64(nullable=True, null_value=-9)) + sentinel.extend(table([(1,), (-9,), (3,)], a=blosc2.int64(nullable=True, null_value=-9))) + + assert from_table["a"].is_null().tolist() == from_rows["a"].is_null().tolist() + assert from_table["a"].is_null().tolist() == sentinel["a"].is_null().tolist() + + +@needs_utf8 +def test_extend_from_a_table_carries_utf8_nulls(): + """The fill is "" here, which is also a legal value -- so it has to be the sidecar.""" + src = table([("x",), (None,), ("",)], s=utf8_spec(null_storage="mask")) + dst = table([], s=utf8_spec(null_storage="mask")) + dst.extend(src) + assert list(dst["s"][:]) == ["x", "", ""] + assert dst["s"].is_null().tolist() == [False, True, False] + + +def test_extend_from_a_table_with_deleted_rows_copies_the_live_ones(): + """Live rows scatter through the physical extent, so a leading slice is the wrong rows.""" + src = simple([10, 11, 12, 13]) + src.delete(0) + dst = simple([]) + dst.extend(src) + assert dst["a"][:].tolist() == [11, 12, 13] + assert dst.nrows == 3 + + +def test_extend_from_a_table_with_holes_keeps_values_and_nulls_aligned(): + src = simple([10, None, 12, None, 14]) + src.delete(0) + dst = simple([]) + dst.extend(src) + assert dst["a"].is_null().tolist() == src["a"].is_null().tolist() + assert dst["a"][:].tolist() == src["a"][:].tolist() + + +def test_extend_from_a_sorted_view_copies_it_in_sorted_order(): + src = simple([3, None, 1, 2]) + dst = simple([]) + dst.extend(src.sort_by("a", view=True)) + # Nulls sort last, in both directions, and the copy has to agree. + assert dst["a"][:].tolist() == [1, 2, 3, 0] + assert dst["a"].is_null().tolist() == [False, False, False, True] + + +def test_extend_from_a_null_free_table_writes_no_sidecar(): + """Decision 9: an absent sidecar is the common state and must stay absent.""" + dst = simple([]) + dst.extend(simple([1, 2, 3])) + assert dst["a"].is_null().tolist() == [False, False, False] + assert dst._null_mask("a") is None From e92982b60e84a84089ecf36681669d1857f27e34 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 10:30:14 +0200 Subject: [PATCH 19/47] Let add_column say a row has no value A mask column can record "missing" without reserving a value, but add_column had no path to it: values=[1, None, 3] and default=None both died in NumPy's astype with "int() argument must be a string, a bytes-like object or a real number, not 'NoneType'", naming neither the column nor a way out. So the one operation that adds a column to a populated table could not be honest about the rows that predate it -- under a sentinel you write the sentinel, under a mask there was nothing to write. _add_column_values now splits nullity out of values= before validating and casting (a null cell has no value to constrain, exactly as extend already argues), a default of None backfills the live rows as null, and the sidecar is written last, once the column and its schema entry are both in place for _ensure_null_mask to pin its row grid. A batch with no null still writes no sidecar. The null-detection rules moved to ctable_nulls.split_batch_validity so add_column and NullChannel.coerce_batch cannot drift: they have to agree on MaskedArray, the NA singletons, NaT, and NaN-is-a-value, but reach it from different directions -- the channel has a live column to ask, add_column only has a spec. fill_item_for is the matching spec-level fill. Two things found on the way: * add_column silently corrupted *every* timestamp column, nullable or not. It cast datetime64 straight to int64 without going through the spec's unit, so a datetime64[s] value landed in a microsecond column and read back as 1970-01-01T00:26:17. extend has always converted correctly; that conversion is now _timestamps_to_stored_int64 and both call it. * A None that genuinely cannot be stored now says which column and what to do instead, from the astype failure path so a well-formed batch pays nothing for the check. Sentinel utf8 is untouched: _coerce maps None onto the sentinel there, and that already worked. tests/ctable: 2203 passed, 1 xfailed. Co-Authored-By: Claude Opus 5 --- src/blosc2/ctable.py | 159 ++++++++++++++++++++++++----- src/blosc2/ctable_nulls.py | 144 +++++++++++++++----------- tests/ctable/test_null_mask_api.py | 132 ++++++++++++++++++++++++ 3 files changed, 350 insertions(+), 85 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 8daecbb33..85bf840a2 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -39,12 +39,15 @@ NULL_NONE, NULL_SENTINEL, NullChannel, + fill_item_for, fill_value_for, + is_na_marker, is_nan_sentinel, kind_of_spec, rewrite_null_predicates, sentinel_guard_expr, sentinel_mask, + split_batch_validity, ) from blosc2.ctable_storage import ( FileTableStorage, @@ -9939,6 +9942,13 @@ def add_column( # noqa: C901 A declared default is still honoured for rows appended later, so *values* and ``blosc2.field(..., default=...)`` can be combined. + For a column whose nulls live in a validity sidecar + (``null_storage="mask"``, the default for ``nullable=True``), a + ``None`` entry writes a real null. Declaring ``default=None`` + likewise backfills every existing row as null, which is the only + way to say "this column has no value for the rows that predate + it" -- a sentinel column has to spell that with its sentinel. + Raises ------ ValueError @@ -9984,8 +9994,20 @@ def add_column( # noqa: C901 raise TypeError( "add_column() does not support dictionary columns; use the constructor with a full schema." ) + # Validity for the live rows, from either the values= batch or a + # default=None backfill. Written once at the end, when the column and + # its schema entry are both in place -- _ensure_null_mask needs both to + # pin the sidecar to the column's own row grid. + live_valid = None + uses_mask = getattr(spec, "uses_mask", False) if values is not None: - values = self._add_column_values(name, compiled_col, values, n_live) + values, live_valid = self._add_column_values(name, compiled_col, values, n_live) + # A declared default of None means "no value for the rows that predate + # this column", which only a sidecar can record; a sentinel column falls + # through to the coercion below and reports it cannot hold a None. + default_is_null = uses_mask and default is not MISSING and is_na_marker(default) + if default_is_null and values is None and n_live > 0: + live_valid = np.zeros(n_live, dtype=np.bool_) if self._is_varlen_scalar_column(compiled_col): # Varlen scalar columns don't use fixed-width NDArray storage, but the @@ -10013,13 +10035,22 @@ def add_column( # noqa: C901 new_col.extend(padded) new_col.flush() else: - if default is not MISSING: + if default_is_null: + # The stored cell is the fill; live_valid above is what records + # that nothing is there to read. + default_val = fill_item_for(spec) + elif default is not MISSING: try: if self._is_ndarray_column(compiled_col): default_val = self._coerce_ndarray_value(name, spec, default) + elif isinstance(spec, timestamp): + # Through the unit, as the values= path above does. + default_val = self._timestamps_to_stored_int64( + spec, [default], np.dtype(spec.dtype) + )[0] else: default_val = spec.dtype.type(default) - except (ValueError, OverflowError) as exc: + except (ValueError, OverflowError, TypeError) as exc: raise TypeError( f"Cannot coerce default {default!r} to dtype {spec.dtype!r}: {exc}" ) from exc @@ -10063,6 +10094,12 @@ def add_column( # noqa: C901 columns=new_columns, columns_by_name={**self._schema.columns_by_name, name: compiled_col}, ) + # Last, because _ensure_null_mask reads both self._cols[name] and the + # schema entry to pin the sidecar to the column's own row grid. A batch + # that held no null leaves live_valid None and writes nothing, so a + # null-free column stays sidecar-free (decision 9). + if live_valid is not None: + self._ensure_null_mask(name)[np.flatnonzero(self._valid_rows[:])] = live_valid if isinstance(self._storage, FileTableStorage): self._storage.save_schema(self._schema_dict_with_computed()) @@ -10074,6 +10111,11 @@ def _varlen_filler(spec, default): just has to be something the spec accepts. """ if default is not MISSING: + # A None default is the caller asking for nulls, not for a literal + # None cell: under mask storage the stored cell is the fill, and the + # sidecar is what says the row is missing. + if getattr(spec, "uses_mask", False) and is_na_marker(default): + return fill_item_for(spec) return default null_value = getattr(spec, "null_value", None) if null_value is not None: @@ -10084,11 +10126,72 @@ def _varlen_filler(spec, default): return "" return None + @staticmethod + def _timestamps_to_stored_int64(spec, values, target_dtype): + """Convert a batch of timestamps to the int64 the column actually stores. + + The conversion has to go **through the spec's unit**: a bare + ``astype(int64)`` on a ``datetime64[s]`` array yields seconds, which a + microsecond column then reads back as a date in 1970. Shared by + ``extend`` and ``add_column``, which both have to store the same + encoding and only one of which used to. + + Values that are already integers pass through, which is what lets a + mask column's fill (``int64.min``, decoded as ``NaT``) survive the + object branch untouched. + """ + arr = np.asarray(values) + if np.issubdtype(arr.dtype, np.datetime64): + return arr.astype(f"datetime64[{spec.unit}]").astype(np.int64) + if arr.dtype.kind in "OUS": + return np.array( + [ + spec.null_value + if v is None + else np.datetime64(v).astype(f"datetime64[{spec.unit}]").astype(np.int64) + if isinstance(v, (np.datetime64, str)) or hasattr(v, "isoformat") + else v + for v in arr + ], + dtype=target_dtype, + ) + return arr + + @staticmethod + def _reject_nulls_this_column_cannot_store(name: str, col: CompiledColumn, values) -> None: + """Say why a ``None`` in ``values=`` cannot be stored, and what to do instead. + + Only a fixed-width column whose nulls live in a sidecar can take a bare + ``None``. Called from the ``astype`` failure path, where NumPy's own + ``int() argument must be ...`` names neither the column nor the way + out; returns quietly when the batch held no null, leaving the caller to + report whatever else went wrong. + """ + _, invalid = split_batch_validity(values, None) + if invalid is None: + return + sentinel = getattr(col.spec, "null_value", None) + remedy = ( + f"write its null_value ({sentinel!r}) instead" + if sentinel is not None + else 'declare the column nullable, e.g. nullable=True (or null_storage="mask")' + ) + raise TypeError( + f"add_column() values= for {name!r} contains a null, which this column cannot store: {remedy}." + ) + def _add_column_values(self, name: str, col: CompiledColumn, values, n_live: int): """Validate and coerce the ``values=`` argument of :meth:`add_column`. - Returns a list for varlen scalar columns (which are fed row by row) and - a dtype-coerced ndarray for the fixed-width ones. + Returns ``(values, valid)``: a list for varlen scalar columns (which are + fed row by row) or a dtype-coerced ndarray for the fixed-width ones, + plus the validity of a mask-storage column whose batch held a null + (``None`` when nothing was, exactly as ``coerce_batch`` reports it). + + Nullity is split out **before** validation and the ``astype`` below, for + the same reason ``extend`` does it: a null cell has no value to + constrain, and ``np.asarray([1, None, 3]).astype(int64)`` raises rather + than yielding anything to store. Constraints declared on the spec are checked here, *before* the ``astype`` below: coercing to a fixed-width dtype truncates a too-long @@ -10097,6 +10200,8 @@ def _add_column_values(self, name: str, col: CompiledColumn, values, n_live: int """ from blosc2.schema_vectorized import validate_column_values + uses_mask = getattr(col.spec, "uses_mask", False) + if self._is_varlen_scalar_column(col): values = list(values) if len(values) != n_live: @@ -10104,8 +10209,20 @@ def _add_column_values(self, name: str, col: CompiledColumn, values, n_live: int f"add_column() values= for {name!r} requires {n_live} entries " f"(live rows), got {len(values)}." ) + valid = None + if uses_mask: + values, valid = split_batch_validity(values, fill_item_for(col.spec)) + values = list(values) + # No null check for the other varlen kinds: the container ones hold + # a native None, and a sentinel utf8 column maps None onto its + # sentinel in _ScalarVarLenArray._coerce. Only the fixed-width + # branch below has nowhere to put one. validate_column_values(col, values) - return values + return values, valid + + valid = None + if uses_mask: + values, valid = split_batch_validity(values, fill_item_for(col.spec)) arr = values[:] if isinstance(values, blosc2.NDArray) else np.asarray(values) if len(arr) != n_live: @@ -10118,9 +10235,17 @@ def _add_column_values(self, name: str, col: CompiledColumn, values, n_live: int f"add_column() values= for {name!r} must have shape {expected}, got {arr.shape}." ) validate_column_values(col, arr) + if isinstance(col.spec, timestamp): + # After validation, which sees the datetimes the caller passed, and + # before the astype, which would otherwise drop the unit. + arr = self._timestamps_to_stored_int64(col.spec, arr, np.dtype(col.spec.dtype)) try: - return arr.astype(col.spec.dtype) - except (ValueError, OverflowError) as exc: + return arr.astype(col.spec.dtype), valid + except (ValueError, OverflowError, TypeError) as exc: + # Only on the failure path, so a well-formed batch never pays for + # the scan: a null is the likeliest reason a batch will not cast, + # and NumPy's own message names neither the column nor the way out. + self._reject_nulls_this_column_cannot_store(name, col, values) raise TypeError( f"Cannot coerce values= for {name!r} to dtype {col.spec.dtype!r}: {exc}" ) from exc @@ -14290,23 +14415,7 @@ def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> if valid is not None: batch_valid[name] = valid if isinstance(col_meta.spec, timestamp): - values = np.asarray(raw_columns[name]) - if np.issubdtype(values.dtype, np.datetime64): - values = values.astype(f"datetime64[{col_meta.spec.unit}]").astype(np.int64) - elif values.dtype.kind in "OUS": - values = np.array( - [ - col_meta.spec.null_value - if v is None - else np.datetime64(v) - .astype(f"datetime64[{col_meta.spec.unit}]") - .astype(np.int64) - if isinstance(v, (np.datetime64, str)) or hasattr(v, "isoformat") - else v - for v in values - ], - dtype=target_dtype, - ) + values = self._timestamps_to_stored_int64(col_meta.spec, raw_columns[name], target_dtype) scalar_processed_cols[name] = np.ascontiguousarray(values, dtype=target_dtype) elif self._is_ndarray_column(col_meta): scalar_processed_cols[name] = self._coerce_ndarray_batch( diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index c43f2684d..efdf01f4c 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -38,6 +38,7 @@ NULL_NONE, NULL_SENTINEL, DictionarySpec, + NDArraySpec, ObjectSpec, StructSpec, VLBytesSpec, @@ -55,6 +56,7 @@ "NULL_NONE", "NULL_SENTINEL", "NullChannel", + "fill_item_for", "fill_value_for", "is_na_marker", "is_nan_sentinel", @@ -63,6 +65,7 @@ "rewrite_null_predicates", "sentinel_guard_expr", "sentinel_mask", + "split_batch_validity", ] # Specs whose cells can hold a native ``None``. ``ListSpec`` is deliberately @@ -121,6 +124,79 @@ def is_na_marker(value) -> builtin_bool: return isinstance(value, np.datetime64) and builtin_bool(np.isnat(value)) +def fill_item_for(spec): + """The whole cell written into one of *spec*'s null slots. + + :func:`~blosc2.schema.fill_value_for` gives the scalar; a fixed-shape + ndarray column's null rows still have to hold something of the right + shape, so it is widened to a full item there. + """ + base = fill_value_for(spec) + if isinstance(spec, NDArraySpec): + return np.full(spec.item_shape, base, dtype=spec.dtype) + return base + + +def split_batch_validity(values, fill): + """Split an incoming batch into ``(storage_values, valid)``. + + *valid* is ``None`` when nothing in the batch was null -- the common case, + and the one that lets a caller skip the sidecar write entirely and so never + materialize one. Otherwise it is a bool array in Arrow polarity (``True`` + = not null), and *storage_values* has *fill* substituted into the null + slots, so what sits under ``valid=False`` is deterministic rather than + whatever NumPy made of a ``None``. + + Null detection follows what the input is able to express: + + * ``np.ma.MaskedArray`` -- ``~arr.mask`` is the validity, verbatim; + * an object array or Python sequence -- ``None`` and the library NA + singletons are null (:func:`is_na_marker`); + * a ``datetime64`` array -- ``NaT`` is null; + * a float array -- **NaN is a value, not a null** (decision 6). A mask + column's whole point is that nullity lives outside the value range, so + nothing in a typed numeric array reads as missing. + + Shared by :meth:`NullChannel.coerce_batch` and ``CTable.add_column``, which + have to agree on all of the above but reach it from different directions: + the channel has a live column to ask, ``add_column`` only has a spec. + """ + if isinstance(values, np.ma.MaskedArray): + valid = ~np.ma.getmaskarray(values) + if valid.ndim > 1: # ndarray column: a row is null only if wholly masked + valid = valid.any(axis=tuple(range(1, valid.ndim))) + filled = np.ma.filled(values, fill) + return filled, (None if valid.all() else valid) + + if isinstance(values, blosc2.NDArray): + # Already in typed storage; there is no way for it to carry a None. + return values, None + + arr = values if isinstance(values, np.ndarray) else np.asarray(values, dtype=object) + if arr.dtype.kind == "M": + invalid = np.isnat(arr) + elif arr.dtype.kind == "O": + # Row-level: for an ndarray column each entry is a whole item, and only + # a wholesale None makes the row null. + invalid = np.fromiter((is_na_marker(v) for v in arr), dtype=np.bool_, count=len(arr)) + else: + # Typed numeric/bool/U/S input has no in-band way to say "null". + return values, None + + if not invalid.any(): + return values, None + out = np.asarray(arr, dtype=object).copy() + if isinstance(fill, np.ndarray): + # An ndarray column's fill is a whole item: assigning it through a + # boolean mask would broadcast its elements across the selected slots + # instead of storing one item in each. + for i in np.flatnonzero(invalid): + out[i] = fill + else: + out[invalid] = fill + return out.tolist(), ~invalid + + def is_nan_sentinel(value) -> bool: """True when *value* is a NaN used as a null sentinel. @@ -445,11 +521,7 @@ def fill_value(self): whole item for a fixed-shape ndarray column, whose null rows still have to hold something of the right shape. """ - base = fill_value_for(self.spec) - col = self._col - if col.is_ndarray: - return np.full(col.item_shape, base, dtype=col.dtype) - return base + return fill_item_for(self.spec) # ------------------------------------------------------------------ # The sidecar (mask kind only) @@ -644,64 +716,16 @@ def coerce_scalar(self, value): return self.fill_value, False def coerce_batch(self, values, n: int): - """Split an incoming batch into ``(storage_values, valid)``. - - *valid* is ``None`` when nothing in the batch was null -- the common - case, and the one that lets the caller skip the sidecar write entirely - and so never materialize one. Otherwise it is a length-*n* bool array - in Arrow polarity (``True`` = not null), and *storage_values* has this - column's fill substituted into the null slots, so what sits under - ``valid=False`` is deterministic rather than whatever NumPy made of a - ``None``. - - Null detection follows what the input is able to express: - - * ``np.ma.MaskedArray`` -- ``~arr.mask`` is the validity, verbatim; - * an object array or Python sequence -- ``None`` and the library NA - singletons are null (:func:`is_na_marker`); - * a ``datetime64`` array -- ``NaT`` is null; - * a float array -- **NaN is a value, not a null** (decision 6). A - mask column's whole point is that nullity lives outside the value - range, so nothing in a typed numeric array reads as missing. + """Split an incoming batch into ``(storage_values, valid)`` for this column. + + Thin wrapper over :func:`split_batch_validity`, which carries the null + detection rules and the reasoning behind them; all this adds is the + column's own fill, and the short-circuit for a column whose nulls do + not live in a sidecar at all. """ if self.kind != NULL_MASK: return values, None - fill = self.fill_value - - if isinstance(values, np.ma.MaskedArray): - valid = ~np.ma.getmaskarray(values) - if valid.ndim > 1: # ndarray column: a row is null only if wholly masked - valid = valid.any(axis=tuple(range(1, valid.ndim))) - filled = np.ma.filled(values, fill) - return filled, (None if valid.all() else valid) - - if isinstance(values, blosc2.NDArray): - # Already in typed storage; there is no way for it to carry a None. - return values, None - - arr = values if isinstance(values, np.ndarray) else np.asarray(values, dtype=object) - if arr.dtype.kind == "M": - invalid = np.isnat(arr) - elif arr.dtype.kind == "O": - # Row-level: for an ndarray column each entry is a whole item, and - # only a wholesale None makes the row null. - invalid = np.fromiter((is_na_marker(v) for v in arr), dtype=np.bool_, count=len(arr)) - else: - # Typed numeric/bool/U/S input has no in-band way to say "null". - return values, None - - if not invalid.any(): - return values, None - out = np.asarray(arr, dtype=object).copy() - if isinstance(fill, np.ndarray): - # An ndarray column's fill is a whole item: assigning it through a - # boolean mask would broadcast its elements across the selected - # slots instead of storing one item in each. - for i in np.flatnonzero(invalid): - out[i] = fill - else: - out[invalid] = fill - return out.tolist(), ~invalid + return split_batch_validity(values, self.fill_value) # ------------------------------------------------------------------ # Lazy predicates over the raw physical array diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index ea3a1aa0b..35b0f2d6c 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -599,3 +599,135 @@ def test_extend_from_a_null_free_table_writes_no_sidecar(): dst.extend(simple([1, 2, 3])) assert dst["a"].is_null().tolist() == [False, False, False] assert dst._null_mask("a") is None + + +# --------------------------------------------------------------------------- +# add_column +# --------------------------------------------------------------------------- +# +# A mask column is the only kind that can say "this row has no value" without +# reserving one, so it is the only way to add a column to a populated table +# and be honest about the rows that predate it. A sentinel column has to +# spell that with its sentinel, and says so. + + +def test_add_column_values_accept_none(): + t = simple([1, 2, 3], spec=blosc2.int64()) + t.add_column("b", blosc2.int64(nullable=True), values=[10, None, 30]) + assert t["b"][:].tolist() == [10, 0, 30] + assert t["b"].is_null().tolist() == [False, True, False] + assert t["b"].null_count() == 1 + + +def test_add_column_default_none_backfills_nulls(): + """The rows that predate the column have no value, and now can say so.""" + t = simple([1, 2, 3], spec=blosc2.int64()) + t.add_column("b", blosc2.field(blosc2.int64(nullable=True), default=None)) + assert t["b"].is_null().tolist() == [True, True, True] + + +def test_add_column_default_none_still_applies_to_later_rows(): + t = simple([1, 2], spec=blosc2.int64()) + t.add_column("b", blosc2.field(blosc2.int64(nullable=True), default=None)) + t.append((3, 9)) + t.extend([(4, None)]) + assert t["b"][:].tolist() == [0, 0, 9, 0] + assert t["b"].is_null().tolist() == [True, True, False, True] + + +def test_add_column_without_a_null_writes_no_sidecar(): + """Decision 9 again: the lazy sidecar must stay lazy here too.""" + t = simple([1, 2, 3], spec=blosc2.int64()) + t.add_column("b", blosc2.int64(nullable=True), values=[10, 20, 30]) + assert t["b"].is_null().tolist() == [False, False, False] + assert t._null_mask("b") is None + + +def test_add_column_scatters_nulls_past_deleted_rows(): + """values= is one entry per *live* row, so validity has to scatter with them.""" + t = simple([0, 1, 2, 3], spec=blosc2.int64()) + t.delete(1) + t.add_column("b", blosc2.int64(nullable=True), values=[10, None, 30]) + assert t["b"].is_null().tolist() == [False, True, False] + t.compact() + assert t["b"].is_null().tolist() == [False, True, False] + assert t["b"][:].tolist() == [10, 0, 30] + + +@needs_utf8 +def test_add_column_utf8_values_accept_none(): + """The fill is "" here, which a genuine row may also hold.""" + t = simple([1, 2, 3], spec=blosc2.int64()) + t.add_column("s", utf8_spec(null_storage="mask"), values=["p", None, ""]) + assert list(t["s"][:]) == ["p", "", ""] + assert t["s"].is_null().tolist() == [False, True, False] + + +def test_add_column_ndarray_values_accept_none(): + t = simple([1, 2], spec=blosc2.int64()) + t.add_column( + "v", + blosc2.ndarray((3,), dtype=blosc2.int64(), nullable=True), + values=[np.array([1, 2, 3]), None], + ) + assert t["v"][:].tolist() == [[1, 2, 3], [0, 0, 0]] + assert t["v"].is_null().tolist() == [False, True] + + +def test_add_column_nulls_survive_a_reopen(tmp_path): + t = simple([1, 2, 3], spec=blosc2.int64()) + urlpath = str(tmp_path / "added.b2t") + t.save(urlpath) + live = blosc2.CTable.open(urlpath, mode="a") + try: + live.add_column("b", blosc2.int64(nullable=True), values=[10, None, 30]) + finally: + live.close() + reopened = blosc2.CTable.open(urlpath) + try: + assert reopened["b"].null_storage == "mask" + assert reopened["b"].is_null().tolist() == [False, True, False] + finally: + reopened.close() + + +@pytest.mark.parametrize( + ("spec", "match"), + [ + (blosc2.int64(), "nullable"), + (blosc2.int64(nullable=True, null_value=-1), r"null_value \(-1\)"), + ], +) +def test_add_column_says_why_a_null_will_not_fit(spec, match): + """Not NumPy's "int() argument must be ...", which names neither the column nor the way out.""" + t = simple([1, 2, 3], spec=blosc2.int64()) + with pytest.raises(TypeError, match=match): + t.add_column("b", spec, values=[10, None, 30]) + + +# A timestamp column stores int64 in the spec's unit, so add_column has to +# convert through that unit exactly as extend does. It did not, which put a +# datetime64[s] value in a microsecond column and read it back as 1970. + + +@pytest.mark.parametrize("nullable", [False, True]) +def test_add_column_timestamps_keep_their_unit(nullable): + when = np.datetime64("2020-01-01T00:00:00", "s") + t = simple([1, 2], spec=blosc2.int64()) + t.add_column("ts", blosc2.timestamp(nullable=nullable), values=[when, when]) + assert t["ts"][:].tolist() == [when.astype("datetime64[us]").item()] * 2 + + +def test_add_column_timestamp_default_keeps_its_unit(): + when = np.datetime64("2020-01-01T00:00:00", "s") + t = simple([1, 2], spec=blosc2.int64()) + t.add_column("ts", blosc2.field(blosc2.timestamp(), default=when)) + assert t["ts"][:].tolist() == [when.astype("datetime64[us]").item()] * 2 + + +def test_add_column_timestamp_null_reads_as_nat(): + when = np.datetime64("2020-01-01T00:00:00", "s") + t = simple([1, 2], spec=blosc2.int64()) + t.add_column("ts", blosc2.timestamp(nullable=True), values=[when, None]) + assert t["ts"].is_null().tolist() == [False, True] + assert np.isnat(t["ts"][:][1]) From f1f120c4e256f6b3b531a869d341f1dad3c7c8ff Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 10:40:23 +0200 Subject: [PATCH 20/47] Check the two null storages against each other Mask storage is a second implementation of nullability, and a second implementation of anything is a standing invitation to drift. The mask-based-nulls plan named a differential oracle its single highest-value test and it was never written; writing it turns "mask is a second implementation" from a permanent liability into a checked invariant. Each case builds the same logical rows twice, once per storage, and asserts the same answer from is_null, notnull, null_count, to_numpy(masked=), fillna, dropna, unique, value_counts, sort_by (both directions, single and multi-key, and as a view), every reduction, argmin/argmax, group_by (as key with dropna both ways, and as value), where (eight expression shapes including OR and negation, indexed and not), and to_arrow -- across every V1 kind. Comparison is always logical: a column is read as its values with None wherever is_null() says so. What sits under a null is the fill for one storage and the sentinel for the other, and decision 5 says that is not part of the format contract, so comparing it would pin down something the design says may change. Three divergences are deliberate and asserted as divergences: NaN is a value under a mask and the null under a sentinel (decision 6), a nullable bool is uint8 only under a sentinel, and complex is mask-only. Two are bugs, pinned as strict xfails so that fixing either trips this suite -- isin() and to_pandas() both read raw values and so match, or emit, whatever stands in for a null. Those xfails are scoped to the kinds where the leak is *visible*: a float fills with NaN and reserves NaN, a timestamp fills with int64.min and reserves it, so for those the two storages leak indistinguishable values and the correct behaviour is asserted outright. Beyond cross-storage agreement, each storage is also checked against itself indexed versus unindexed -- agreement alone would not catch an index that leaks nulls into both storages the same way. Verified non-vacuous by mutation: sorting mask nulls first fails 30 cases, under-reporting null_count fails 10, and withdrawing the validity guard from queries (the plan's Addendum 2 bug) fails 42. tests/ctable: 2470 passed, 15 xfailed. Co-Authored-By: Claude Opus 5 --- tests/ctable/test_null_storage_equivalence.py | 554 ++++++++++++++++++ 1 file changed, 554 insertions(+) create mode 100644 tests/ctable/test_null_storage_equivalence.py diff --git a/tests/ctable/test_null_storage_equivalence.py b/tests/ctable/test_null_storage_equivalence.py new file mode 100644 index 000000000..d7b4a6c7e --- /dev/null +++ b/tests/ctable/test_null_storage_equivalence.py @@ -0,0 +1,554 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""The differential oracle: sentinel and mask storage must answer alike. + +CTable can keep a column's nulls in two places -- an in-band **sentinel**, or a +sidecar **validity mask** -- and a second implementation of anything is a +standing invitation to drift. This suite builds the *same logical data* twice, +once each way, and asserts that every public API gives the same answer. That +turns "mask is a second implementation" from a permanent liability into a +checked invariant. + +Comparison is always **logical**, never physical: a column is read as its +values with ``None`` substituted wherever ``is_null()`` is True (:func:`logical` +below). What sits underneath a null is the fill for one storage and the +sentinel for the other, and neither is part of the format contract -- asserting +on it would pin down something the design explicitly says may change. + +Three differences are **deliberate** and asserted as differences rather than +papered over: + +* **NaN is a value under mask storage** and null under a sentinel one + (decision 6). This is the entire point of a side channel, so the float cases + here carry no NaN, and :func:`test_nan_is_a_value_only_under_mask_storage` + pins the divergence on its own. +* **A nullable bool is physically ``uint8``** under a sentinel, to leave room + for the reserved ``255``, and plain ``np.bool_`` under a mask. The values + still compare equal (``True == 1``), which is what the oracle checks. +* **complex is mask-only**: no complex value is safe to reserve, so there is no + sentinel column to compare against. + +Two more differences are **not** deliberate; they are open bugs, pinned below +as strict xfails so that fixing either one trips this suite. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest +from utf8_compat import HAVE_UTF8, needs_utf8, utf8_spec + +import blosc2 + +T0 = np.datetime64("2021-03-04T05:06:07", "s") + + +def secs(n: int) -> np.timedelta64: + """A timedelta with an explicit unit; a bare int is deprecated in NumPy.""" + return np.timedelta64(n, "s") + + +#: ``kind -> (spec factory, logical values, sentinel literal)``. +#: +#: Every V1 kind, and the values are chosen to be awkward on purpose: a full +#: range of ``int8``/``uint8``, a genuine ``""`` beside a null in each text +#: kind, and a ``0``/``0.0`` that a fill could be confused with. No float NaN +#: -- that is decision 6's business and is tested separately. +KINDS: dict = { + "int64": (lambda **k: blosc2.int64(**k), [5, None, -3, 0, 7], -9), + "int8": (lambda **k: blosc2.int8(**k), [1, None, -5, 0, 127], -128), + "uint8": (lambda **k: blosc2.uint8(**k), [1, None, 200, 0, 7], 255), + "float64": (lambda **k: blosc2.float64(**k), [1.5, None, -2.5, 0.0, 7.25], float("nan")), + "float32": (lambda **k: blosc2.float32(**k), [1.5, None, -2.5, 0.0, 7.25], float("nan")), + "bool": (lambda **k: blosc2.bool(**k), [True, None, False, True, False], 255), + "timestamp": ( + lambda **k: blosc2.timestamp(**k), + [T0, None, T0 + secs(60), T0 + secs(5), T0 + secs(1)], + int(np.iinfo(np.int64).min), + ), + "string": (lambda **k: blosc2.string(max_length=8, **k), ["a", None, "", "zz", "m"], "ZZZZZZZZ"), + "bytes": ( + lambda **k: blosc2.bytes(max_length=8, **k), + [b"a", None, b"", b"zz", b"m"], + b"ZZZZZZZZ", + ), +} +if HAVE_UTF8: + KINDS["utf8"] = (utf8_spec, ["a", None, "", "zz", "m"], "__BLOSC2_NULL__") + +ALL_KINDS = sorted(KINDS) +#: Kinds whose values support ordered arithmetic reductions. +NUMERIC_KINDS = [k for k in ALL_KINDS if k.startswith(("int", "uint", "float"))] +#: The row that is null in every table this module builds. +NULL_ROW = 1 + +#: Kinds whose *fill* and *sentinel* happen to be the same value, so an API +#: that leaks what stands in for a null leaks something indistinguishable +#: either way. A float column fills with NaN and reserves NaN; a timestamp +#: fills with ``int64.min`` and reserves ``int64.min``. They are the kinds +#: where the two bugs below are invisible -- not the kinds where they are +#: fixed -- so they still assert the correct behaviour, just without the xfail. +INDISTINGUISHABLE_FILL = ("float32", "float64", "timestamp") + + +def leaks_its_fill(kind: str) -> bool: + """Whether a raw read of *kind*'s null slot exposes an ordinary-looking value.""" + return kind not in INDISTINGUISHABLE_FILL + + +def kinds_xfailing_on_leak(reason: str): + """``ALL_KINDS``, with the fill-leaking ones marked ``xfail(strict=True)``.""" + return [ + pytest.param( + kind, + marks=pytest.mark.xfail(strict=True, reason=reason) if leaks_its_fill(kind) else (), + ) + for kind in ALL_KINDS + ] + + +def annotation_for(spec): + if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)): + return object + return spec.python_type + + +def build(kind: str, storage: str, *, capacity: int = 64): + """The same logical rows, stored *storage*'s way. + + Column ``a`` carries the nulls; ``g`` is a plain non-nullable key so the + group-by and multi-key sort cases have something to work with. + """ + factory, values, sentinel = KINDS[kind] + if storage == "mask": + spec = factory(nullable=True, null_storage="mask") + cells = values + else: + spec = factory(nullable=True, null_value=sentinel) + # A sentinel column has no way to spell "null" other than its sentinel. + cells = [sentinel if v is None else v for v in values] + row_cls = dataclasses.make_dataclass( + "EquivRow", + [ + ("a", annotation_for(spec), blosc2.field(spec)), + ("g", str, blosc2.field(blosc2.string(max_length=4))), + ], + ) + t = blosc2.CTable(row_cls, expected_size=capacity) + t.extend([(v, f"g{i % 2}") for i, v in enumerate(cells)]) + return t + + +def both(kind: str, **kwargs): + """The mask-backed and sentinel-backed tables for *kind*.""" + return build(kind, "mask", **kwargs), build(kind, "sentinel", **kwargs) + + +def logical(col) -> list: + """A column as its observable contents: values, with ``None`` where null. + + This is the whole comparison discipline of this suite. Reading ``col[:]`` + alone would compare the fill against the sentinel and fail everywhere for + reasons that are not bugs. + """ + nulls = col.is_null() + out = [] + for value, is_null in zip(col[:], nulls, strict=True): + if is_null: + out.append(None) + else: + out.append(value.item() if hasattr(value, "item") else value) + return out + + +def logical_rows(t, cols=("a", "g")) -> list[tuple]: + """Every live row of *t*, each column read through :func:`logical`.""" + columns = [logical(t[c]) for c in cols] + return [tuple(col[i] for col in columns) for i in range(t.nrows)] + + +def _null_last(value): + """Sort key putting ``None`` last and ordering the rest by value. + + By value, not by ``repr``: a nullable bool is ``np.bool_`` under a mask and + ``uint8`` under a sentinel, so ``False``/``0`` sort together here and + compare equal afterwards, while their reprs would not. + """ + return (value is None, 0 if value is None else value) + + +def assert_same(mask_result, sentinel_result, what: str) -> None: + """Assert the two storages agree, reporting which API disagreed.""" + assert mask_result == sentinel_result, ( + f"{what}: mask storage gave {mask_result!r}, sentinel storage gave {sentinel_result!r}" + ) + + +# --------------------------------------------------------------------------- +# The null API +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_reads_agree(kind): + """The baseline: the same rows, and the same rows are null.""" + m, s = both(kind) + assert_same(logical(m["a"]), logical(s["a"]), "column contents") + assert_same(logical(m["a"])[NULL_ROW], None, "the null row") + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_null_api_agrees(kind): + m, s = both(kind) + assert_same(m["a"].is_null().tolist(), s["a"].is_null().tolist(), "is_null") + assert_same(m["a"].notnull().tolist(), s["a"].notnull().tolist(), "notnull") + assert_same(m["a"].null_count(), s["a"].null_count(), "null_count") + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_to_numpy_masked_agrees(kind): + """The masked view is the one read that must work for *both* storages.""" + m, s = both(kind) + got, want = m["a"].to_numpy(masked=True), s["a"].to_numpy(masked=True) + assert_same(got.mask.tolist(), want.mask.tolist(), "to_numpy(masked=True).mask") + assert_same(got.compressed().tolist(), want.compressed().tolist(), "to_numpy(masked=True) values") + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_dropna_agrees(kind): + m, s = both(kind) + assert_same(logical_rows(m.dropna()), logical_rows(s.dropna()), "dropna") + assert m.dropna().nrows == len(KINDS[kind][1]) - 1 + + +@pytest.mark.parametrize("kind", NUMERIC_KINDS) +def test_fillna_agrees(kind): + m, s = both(kind) + assert_same(m["a"].fillna(42).tolist(), s["a"].fillna(42).tolist(), "fillna") + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_unique_and_value_counts_agree(kind): + """Both exclude the null, and neither leaks what sits under it.""" + m, s = both(kind) + assert_same(sorted(m["a"].unique().tolist()), sorted(s["a"].unique().tolist()), "unique") + assert_same( + sorted(m["a"].value_counts().items()), + sorted(s["a"].value_counts().items()), + "value_counts", + ) + + +# --------------------------------------------------------------------------- +# Ordering +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", ALL_KINDS) +@pytest.mark.parametrize("ascending", [True, False]) +def test_sort_by_agrees(kind, ascending): + """Nulls sort last in both directions, whichever storage they live in.""" + m, s = both(kind) + got = logical(m.sort_by("a", ascending=ascending)["a"]) + want = logical(s.sort_by("a", ascending=ascending)["a"]) + assert_same(got, want, f"sort_by(ascending={ascending})") + assert got[-1] is None, "nulls sort last" + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_multi_key_sort_agrees(kind): + m, s = both(kind) + assert_same(logical_rows(m.sort_by(["g", "a"])), logical_rows(s.sort_by(["g", "a"])), "sort_by 2 keys") + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_sorted_view_agrees(kind): + m, s = both(kind) + assert_same( + logical(m.sort_by("a", view=True)["a"]), + logical(s.sort_by("a", view=True)["a"]), + "sort_by(view=True)", + ) + + +# --------------------------------------------------------------------------- +# Reductions +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", NUMERIC_KINDS) +@pytest.mark.parametrize("op", ["sum", "mean", "min", "max", "std"]) +def test_reductions_agree(kind, op): + """Every reduction skips the null rather than folding in what stands for it.""" + m, s = both(kind) + got, want = getattr(m["a"], op)(), getattr(s["a"], op)() + assert got == pytest.approx(want), f"{op}: {got!r} != {want!r}" + + +@pytest.mark.parametrize("kind", NUMERIC_KINDS + ["timestamp"]) +@pytest.mark.parametrize("op", ["argmin", "argmax"]) +def test_arg_reductions_agree(kind, op): + m, s = both(kind) + got, want = int(getattr(m["a"], op)()), int(getattr(s["a"], op)()) + assert_same(got, want, op) + assert got != NULL_ROW, "a null row can never be the extremum" + + +@pytest.mark.parametrize("kind", ["timestamp", "string", "bytes"]) +@pytest.mark.parametrize("op", ["min", "max"]) +def test_ordered_non_numeric_reductions_agree(kind, op): + m, s = both(kind) + assert_same(getattr(m["a"], op)(), getattr(s["a"], op)(), op) + + +# --------------------------------------------------------------------------- +# Grouping +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", ALL_KINDS) +@pytest.mark.parametrize("dropna", [True, False]) +def test_group_by_key_agrees(kind, dropna): + """As a *key*: the null forms its own group, keyed None, under both storages. + + A sentinel column stores that group's key as its sentinel and a mask column + as its fill, so this only holds when the key column is read logically -- + which is the point. It is also the one place a sentinel column has to + round-trip its own sentinel back into a null. + """ + m, s = both(kind) + got = sorted(logical(m.group_by("a", dropna=dropna).count("g")["a"]), key=_null_last) + want = sorted(logical(s.group_by("a", dropna=dropna).count("g")["a"]), key=_null_last) + assert_same(got, want, f"group_by key (dropna={dropna})") + assert (None in got) is not dropna, "dropna decides whether the null group exists" + + +@pytest.mark.parametrize("kind", NUMERIC_KINDS) +@pytest.mark.parametrize("op", ["sum", "min", "max", "count"]) +def test_group_by_value_agrees(kind, op): + """As a *value*: the null must not be aggregated as if it were its fill.""" + m, s = both(kind) + got = getattr(m.group_by("g"), op)("a") + want = getattr(s.group_by("g"), op)("a") + assert_same(logical_rows(got, got.col_names), logical_rows(want, want.col_names), f"group_by {op}") + + +# --------------------------------------------------------------------------- +# Queries +# --------------------------------------------------------------------------- + +#: Expression shapes the two storages must answer identically. Each is written +#: so the *stored* null value would satisfy it under at least one storage if +#: nullity leaked -- which is what makes them worth asserting. +NUMERIC_EXPRESSIONS = [ + "a > 0", + "a < 3", + "a == 0", + "a != 0", + "(a > 0) & (g == 'g0')", + "(a > 0) | (g == 'g1')", + "~(a > 0)", + "(a < 3) & (a > -100)", +] + + +@pytest.mark.parametrize("kind", ["int64", "int8", "uint8", "float64"]) +@pytest.mark.parametrize("expression", NUMERIC_EXPRESSIONS) +def test_where_agrees(kind, expression): + m, s = both(kind) + assert_same(logical_rows(m.where(expression)), logical_rows(s.where(expression)), f"where({expression})") + + +def build_big(storage: str, *, indexed: bool, n: int = 4000): + """A table big enough for the planner to actually reach for an index. + + Every seventh row is null, and the values straddle zero so the expressions + below select a real range rather than everything or nothing. + """ + values = [None if i % 7 == 0 else (i % 101) - 50 for i in range(n)] + sentinel = -9999 + if storage == "mask": + spec = blosc2.int64(nullable=True, null_storage="mask") + cells = values + else: + spec = blosc2.int64(nullable=True, null_value=sentinel) + cells = [sentinel if v is None else v for v in values] + row_cls = dataclasses.make_dataclass( + "BigRow", + [("a", int, blosc2.field(spec)), ("g", str, blosc2.field(blosc2.string(max_length=4)))], + ) + t = blosc2.CTable(row_cls, expected_size=n + 16, create_summary_index=False) + t.extend([(v, f"g{i % 2}") for i, v in enumerate(cells)]) + if indexed: + t.create_index("a", kind="summary") + assert "a" in t._get_index_catalog(), "the index this test is about was not built" + else: + assert "a" not in t._get_index_catalog(), "this table was meant to have no index" + return t + + +@pytest.mark.parametrize("expression", NUMERIC_EXPRESSIONS) +def test_indexed_where_agrees(expression): + """The same answers with an index in play. + + An ordered index answers by slicing the sorted column rather than by + evaluating the predicate, so this is the path where a leaked null shows up + as a *different* result from the identical unindexed query -- the failure + the plan's Addendum 2 found leaking every null through a float index. + """ + m = build_big("mask", indexed=True) + s = build_big("sentinel", indexed=True) + assert_same( + logical_rows(m.where(expression)), logical_rows(s.where(expression)), f"indexed {expression}" + ) + + +@pytest.mark.parametrize("storage", ["mask", "sentinel"]) +@pytest.mark.parametrize("expression", NUMERIC_EXPRESSIONS) +def test_an_index_does_not_change_the_answer(storage, expression): + """Each storage must agree with *itself*, indexed versus not. + + Cross-storage agreement alone would not catch an index that leaks nulls + into both storages the same way, so this is the other half of the oracle: + the scan is the reference, and the index has to match it. + """ + indexed = build_big(storage, indexed=True) + scanned = build_big(storage, indexed=False) + assert_same( + logical_rows(indexed.where(expression)), + logical_rows(scanned.where(expression)), + f"{storage}: indexed vs scanned {expression}", + ) + + +@needs_utf8 +@pytest.mark.parametrize("expression", ["a == 'a'", "a != 'a'", "(a == 'a') | (g == 'g1')"]) +def test_text_where_agrees(expression): + m, s = both("utf8") + assert_same(logical_rows(m.where(expression)), logical_rows(s.where(expression)), f"where({expression})") + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_a_null_never_satisfies_a_predicate(kind): + """SQL WHERE semantics, which is what both storages promise (decision 8).""" + for t in both(kind): + for row in logical_rows(t.where("a == a")): + assert row[0] is not None + + +# --------------------------------------------------------------------------- +# Export +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_to_arrow_agrees(kind): + """Arrow has a validity bitmap of its own, so this is the exact comparison.""" + m, s = both(kind) + got, want = m.to_arrow()["a"], s.to_arrow()["a"] + assert_same(got.null_count, want.null_count, "to_arrow null_count") + assert_same(got.to_pylist(), want.to_pylist(), "to_arrow values") + assert got.to_pylist()[NULL_ROW] is None + + +# --------------------------------------------------------------------------- +# Deliberate divergences +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", ["float64", "float32"]) +def test_nan_is_a_value_only_under_mask_storage(kind): + """Decision 6, and the reason the float rows above carry no NaN. + + A sentinel float column spells its null ``NaN``, so it cannot also hold one + as data; a mask column follows Arrow and keeps NaN an ordinary value. This + is the divergence the side channel exists to create, so it is asserted + rather than tolerated. + """ + factory = KINDS[kind][0] + nan = float("nan") + + mask_t = build_one(factory(nullable=True, null_storage="mask"), [1.0, nan, 3.0]) + sent_t = build_one(factory(nullable=True, null_value=nan), [1.0, nan, 3.0]) + + assert mask_t["a"].is_null().tolist() == [False, False, False], "NaN is a value under a mask" + assert sent_t["a"].is_null().tolist() == [False, True, False], "NaN is the null under a sentinel" + assert mask_t["a"].null_count() == 0 + assert sent_t["a"].null_count() == 1 + # And it follows through to the reductions, which is where a user meets it. + assert np.isnan(mask_t["a"].sum()) + assert sent_t["a"].sum() == 4.0 + + +def build_one(spec, values): + """A one-column table of *values*, with no null coercion of any kind.""" + row_cls = dataclasses.make_dataclass("NanRow", [("a", annotation_for(spec), blosc2.field(spec))]) + t = blosc2.CTable(row_cls, expected_size=16) + t.extend([(v,) for v in values]) + return t + + +def test_nullable_bool_is_uint8_only_under_a_sentinel(): + """The reserved 255 needs room; a mask column has np.bool_ and needs none.""" + m, s = both("bool") + assert m["a"].dtype == np.dtype(np.bool_) + assert s["a"].dtype == np.dtype(np.uint8) + # The physical difference is invisible to the logical read, which is why + # every other bool case in this suite compares equal. + assert_same(logical(m["a"]), logical(s["a"]), "bool contents") + + +def test_complex_has_no_sentinel_to_compare_against(): + """complex is mask-only: no complex value is safe to reserve.""" + with pytest.raises((TypeError, ValueError)): + blosc2.complex128(nullable=True, null_value=0j) + assert blosc2.complex128(nullable=True).null_storage == "mask" + + +# --------------------------------------------------------------------------- +# Divergences that are bugs +# --------------------------------------------------------------------------- + + +ISIN_LEAK = ( + "isin() reads col[:] and tests membership on the raw values, so it matches whatever " + "stands in for a null -- the fill under mask storage, the sentinel under a sentinel " + "one. A null row should match nothing." +) +TO_PANDAS_LEAK = ( + "to_pandas() writes the raw values, so a null arrives as the fill under mask storage " + "and as the sentinel under a sentinel one. Neither is NA, and to_arrow() on the same " + "data is already exact." +) + + +@pytest.mark.parametrize("kind", kinds_xfailing_on_leak(ISIN_LEAK)) +def test_isin_agrees(kind): + m, s = both(kind) + # Probe with each storage's own stand-in for a null: neither should match. + mask_stand_in = m["a"][:][NULL_ROW] + sentinel_stand_in = s["a"][:][NULL_ROW] + for probe in (mask_stand_in, sentinel_stand_in): + got, want = m["a"].isin([probe]).tolist(), s["a"].isin([probe]).tolist() + assert_same(got, want, f"isin([{probe!r}])") + assert not got[NULL_ROW], "a null row is not a member of anything" + + +@pytest.mark.parametrize("kind", kinds_xfailing_on_leak(TO_PANDAS_LEAK)) +def test_to_pandas_agrees(kind): + pd = pytest.importorskip("pandas") + m, s = both(kind) + got, want = m.to_pandas()["a"], s.to_pandas()["a"] + assert_same(got.isna().tolist(), want.isna().tolist(), "to_pandas isna") + assert got.isna()[NULL_ROW], "the null row should read as NA" + + +if __name__ == "__main__": + pytest.main(["-v", __file__]) From 8c217a7d909f337fe9e09632e02b4dd6b87857fd Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 11:01:36 +0200 Subject: [PATCH 21/47] Stop isin() matching what stands in for a null isin() read col[:] and tested membership on the raw values, so a null row matched whatever occupies its slot: the fill under mask storage, the sentinel under a sentinel one. Neither is the row's value -- the fill is explicitly not part of the format contract, and a sentinel is a value the column promises never to mean literally -- so col.isin([0]) returned True for a null in an int column that holds no zero, and isin([""]) did the same for text. Membership is now asked of the row's value, and a null row has none, so it matches nothing. Nullity comes from the null channel, which makes this right for both storages and for a view's own rows at the same time. None in the values is how you select the nulls, matching exactly what is_null() reports; pandas.NA and NaT are accepted as spellings of the same request. That is what dictionary columns have always done with their reserved code, so the three kinds now answer alike rather than two of them disagreeing. A float NaN is deliberately not a null marker: under mask storage NaN is an ordinary value (decision 6). The equivalence oracle's isin xfail comes off, and two cases join it there -- None selecting exactly the nulls, and None beside a real value getting both. One wrinkle worth recording: a timestamp's fill decodes to NaT, which *is* a way of spelling missing, so probing with it legitimately selects the nulls rather than nothing. Left alone, as a separate concern: isin([float("nan")]) does not match a genuine NaN, because Python set membership compares NaN unequal to itself. That predates this change and has nothing to do with nulls. tests/ctable: 2505 passed, 8 xfailed. Co-Authored-By: Claude Opus 5 --- src/blosc2/ctable.py | 28 ++++++++- tests/ctable/test_null_mask_api.py | 59 +++++++++++++++++ tests/ctable/test_null_storage_equivalence.py | 63 ++++++++++++------- 3 files changed, 126 insertions(+), 24 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 85bf840a2..304ae936c 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -2508,6 +2508,19 @@ def _dictionary_eq(self, other, *, negate: bool = False): def isin(self, values) -> np.ndarray: """Return a boolean array True where the live value is in *values*. + A **null row matches nothing**, whichever way this column stores its + nulls. Testing membership against the raw values would instead match + whatever stands in for a null -- the fill under mask storage, the + sentinel under a sentinel one -- and those are not the row's value: + the fill is explicitly not part of the format contract, and a sentinel + is a reserved value the column promises never to mean literally. + + To select the nulls, put ``None`` in *values*: it matches exactly the + rows :meth:`is_null` reports, and nothing else. ``pandas.NA`` and a + ``NaT`` are accepted as spellings of the same request. A float + ``NaN`` is **not** one of them -- under mask storage NaN is an ordinary + value (decision 6 of the mask-nulls design), so it is matched as data. + For dictionary columns this performs efficient integer-code membership testing (no decoding of all values). Values absent from the dictionary are treated as not-present. @@ -2516,12 +2529,21 @@ def isin(self, values) -> np.ndarray: membership in a set. """ if self.is_dictionary: + # Its own path already answers None with the reserved null code. return self._dictionary_isin(values) + values = list(values) + wants_null = any(is_na_marker(v) for v in values) + test_set = {v for v in values if not is_na_marker(v)} + live_values = self[:] - test_set = set(values) if isinstance(live_values, np.ndarray): - return np.array([v in test_set for v in live_values.tolist()], dtype=bool) - return np.array([v in test_set for v in live_values], dtype=bool) + live_values = live_values.tolist() + found = np.array([v in test_set for v in live_values], dtype=bool) + + nulls = self._nulls.null_mask() + if nulls.any(): + found[nulls] = wants_null + return found def _dictionary_isin(self, values) -> np.ndarray: """Return a boolean array for in-membership tests against a dictionary column.""" diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index 35b0f2d6c..836f6379b 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -731,3 +731,62 @@ def test_add_column_timestamp_null_reads_as_nat(): t.add_column("ts", blosc2.timestamp(nullable=True), values=[when, None]) assert t["ts"].is_null().tolist() == [False, True] assert np.isnat(t["ts"][:][1]) + + +# --------------------------------------------------------------------------- +# isin +# --------------------------------------------------------------------------- +# +# Membership is asked of the row's *value*, and a null row has none. Testing +# the raw values instead matched whatever stands in for a null -- the fill +# here, a sentinel elsewhere -- neither of which the column ever means +# literally. Cross-storage agreement is pinned in +# test_null_storage_equivalence.py; these are the mask-only corners. + + +def test_isin_does_not_match_the_fill(): + t = simple([1, None, 0]) + # Rows 1 and 2 both hold a physical 0; only row 2 holds it as a value. + assert t["a"][:].tolist() == [1, 0, 0] + assert t["a"].isin([0]).tolist() == [False, False, True] + + +def test_isin_none_selects_the_nulls(): + t = simple([1, None, 3, None]) + assert t["a"].isin([None]).tolist() == t["a"].is_null().tolist() + + +def test_isin_none_can_be_combined_with_values(): + t = simple([1, None, 3]) + assert t["a"].isin([3, None]).tolist() == [False, True, True] + + +def test_isin_pandas_na_spells_the_same_request(): + pd = pytest.importorskip("pandas") + t = simple([1, None, 3]) + assert t["a"].isin([pd.NA]).tolist() == [False, True, False] + + +@needs_utf8 +def test_isin_does_not_match_the_empty_string_fill(): + """The utf8 fill is "", which a genuine row may hold -- so it has to be the sidecar.""" + t = table([("x",), (None,), ("",)], s=utf8_spec(null_storage="mask")) + assert list(t["s"][:]) == ["x", "", ""] + assert t["s"].isin([""]).tolist() == [False, False, True] + + +def test_isin_on_a_view_uses_the_views_rows(): + t = simple([1, None, 3, None, 5]) + assert t.sort_by("a", view=True)["a"].isin([None]).tolist() == [False, False, False, True, True] + assert t.take([1, 2])["a"].isin([None]).tolist() == [True, False] + + +def test_isin_keeps_nan_a_value(): + """Decision 6: only mask=False is missing, so a NaN row is not a null row.""" + t = simple([1.0, float("nan"), None], spec=blosc2.float64(null_storage="mask")) + assert t["a"].is_null().tolist() == [False, False, True] + assert t["a"].isin([None]).tolist() == [False, False, True] + + +def test_isin_on_an_empty_column(): + assert simple([])["a"].isin([1, None]).tolist() == [] diff --git a/tests/ctable/test_null_storage_equivalence.py b/tests/ctable/test_null_storage_equivalence.py index d7b4a6c7e..3bf1e6906 100644 --- a/tests/ctable/test_null_storage_equivalence.py +++ b/tests/ctable/test_null_storage_equivalence.py @@ -33,8 +33,9 @@ * **complex is mask-only**: no complex value is safe to reserve, so there is no sentinel column to compare against. -Two more differences are **not** deliberate; they are open bugs, pinned below -as strict xfails so that fixing either one trips this suite. +One more difference is **not** deliberate -- ``to_pandas`` emits whatever +stands in for a null instead of NA -- and is pinned below as a strict xfail, so +that fixing it trips this suite and the marker comes off. """ from __future__ import annotations @@ -46,6 +47,7 @@ from utf8_compat import HAVE_UTF8, needs_utf8, utf8_spec import blosc2 +from blosc2.ctable_nulls import is_na_marker T0 = np.datetime64("2021-03-04T05:06:07", "s") @@ -93,8 +95,8 @@ def secs(n: int) -> np.timedelta64: #: that leaks what stands in for a null leaks something indistinguishable #: either way. A float column fills with NaN and reserves NaN; a timestamp #: fills with ``int64.min`` and reserves ``int64.min``. They are the kinds -#: where the two bugs below are invisible -- not the kinds where they are -#: fixed -- so they still assert the correct behaviour, just without the xfail. +#: where the bug below is invisible -- not the kinds where it is fixed -- so +#: they still assert the correct behaviour, just without the xfail. INDISTINGUISHABLE_FILL = ("float32", "float64", "timestamp") @@ -512,16 +514,47 @@ def test_complex_has_no_sentinel_to_compare_against(): assert blosc2.complex128(nullable=True).null_storage == "mask" +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_isin_agrees(kind): + """A null matches nothing -- not even the value that stands in for it.""" + m, s = both(kind) + # Probe with each storage's own stand-in for a null. Neither should match, + # because neither stand-in is the row's value: the fill is not part of the + # format contract, and the sentinel is reserved. + for probe in (m["a"][:][NULL_ROW], s["a"][:][NULL_ROW]): + got, want = m["a"].isin([probe]).tolist(), s["a"].isin([probe]).tolist() + assert_same(got, want, f"isin([{probe!r}])") + if is_na_marker(probe): + # A timestamp's fill decodes to NaT, which *is* a way of spelling + # "missing", so probing with it is asking for the nulls. + assert got[NULL_ROW], f"{probe!r} is a null marker, so it selects nulls" + else: + assert not got[NULL_ROW], "a null row is not a member of anything" + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_isin_none_selects_the_nulls(kind): + """``None`` is how you ask for them, and it agrees with is_null exactly.""" + for t in both(kind): + assert_same(t["a"].isin([None]).tolist(), t["a"].is_null().tolist(), "isin([None]) vs is_null") + + +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_isin_none_beside_a_real_value(kind): + """Asking for a value *and* the nulls gets both, and nothing else.""" + m, s = both(kind) + rows = logical(m["a"]) + present = rows[0] + want = [v is None or v == present for v in rows] + assert m["a"].isin([present, None]).tolist() == want + assert s["a"].isin([present, None]).tolist() == want + + # --------------------------------------------------------------------------- # Divergences that are bugs # --------------------------------------------------------------------------- -ISIN_LEAK = ( - "isin() reads col[:] and tests membership on the raw values, so it matches whatever " - "stands in for a null -- the fill under mask storage, the sentinel under a sentinel " - "one. A null row should match nothing." -) TO_PANDAS_LEAK = ( "to_pandas() writes the raw values, so a null arrives as the fill under mask storage " "and as the sentinel under a sentinel one. Neither is NA, and to_arrow() on the same " @@ -529,18 +562,6 @@ def test_complex_has_no_sentinel_to_compare_against(): ) -@pytest.mark.parametrize("kind", kinds_xfailing_on_leak(ISIN_LEAK)) -def test_isin_agrees(kind): - m, s = both(kind) - # Probe with each storage's own stand-in for a null: neither should match. - mask_stand_in = m["a"][:][NULL_ROW] - sentinel_stand_in = s["a"][:][NULL_ROW] - for probe in (mask_stand_in, sentinel_stand_in): - got, want = m["a"].isin([probe]).tolist(), s["a"].isin([probe]).tolist() - assert_same(got, want, f"isin([{probe!r}])") - assert not got[NULL_ROW], "a null row is not a member of anything" - - @pytest.mark.parametrize("kind", kinds_xfailing_on_leak(TO_PANDAS_LEAK)) def test_to_pandas_agrees(kind): pd = pytest.importorskip("pandas") From c53a76fb9561e6419b2564fb741b92ad3d01c41e Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 11:11:10 +0200 Subject: [PATCH 22/47] Send nulls to pandas as NA to_pandas() wrote the raw values, so a null arrived as whatever occupies its slot -- the fill under mask storage, the sentinel under a sentinel one. A nullable int column came out holding 0 or -9 as ordinary data, where to_arrow() on the same table was already exact. A null now becomes pandas NA, and a column holding one is given a dtype that can say so: Int64/UInt8/... for integers, boolean for bools (both storages, since a sentinel bool is only physically uint8), NaN/NaT where the NumPy dtype already has one, and object with None for text, bytes, complex and ndarray cells. A column with no null is returned untouched, so no dtype moves under anyone; only a column that could not previously be represented changes, which is the same data-dependent widening pyarrow.Table.to_pandas does and for the same reason. The dictionary and variable-length kinds already materialised their nulls as None and are left alone. from_pandas had to keep up, or a table would not have survived its own round trip: to_numpy(dtype=...) raises on every NA-carrying extension dtype to_pandas now emits. It stays the fast path, and a series holding a missing value falls back to handing cells over one at a time for the write path to split, with a sentinel column taking its sentinel. That fallback also fixes two silent corruptions that predate this work: a missing cell bound for a bytes or string column was coerced by to_numpy() into the *string* b'None' / 'nan' rather than rejected. Verified: 18 of the 20 kind/storage combinations now round-trip through pandas losslessly. The two that do not are mask-backed float, and that is a limit of the destination -- pandas has no float dtype separating NaN from missing, since even Float64 folds a NaN into NA -- so it is documented on to_pandas, pinned by a test, and pointed at to_arrow. The equivalence oracle's last xfail comes off; it now passes clean, with the pandas round trip added to it. Re-checked non-vacuous by mutation: reverting to_pandas to raw values fails 14 of its cases. tests/ctable: 2542 passed, 1 xfailed (the unrelated operator-form negation leak). Co-Authored-By: Claude Opus 5 --- src/blosc2/ctable.py | 137 ++++++++++++++++- tests/ctable/test_null_storage_equivalence.py | 140 +++++++++++++----- 2 files changed, 231 insertions(+), 46 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 304ae936c..e1a2579fa 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -9711,6 +9711,22 @@ def to_pandas(self): columns become ``object``-dtype columns whose cells hold NumPy arrays of per-row shape *item_shape*. + A **null becomes pandas NA**, never the value that stands in for it in + storage. A column holding one is therefore given a dtype that can say + so -- ``Int64``/``UInt8``/… for integers, ``boolean`` for bools, + ``NaN``/``NaT`` where the NumPy dtype already has one, and ``object`` + with ``None`` for text, bytes, complex and ndarray cells. A column + with **no** null keeps the dtype it has always had here. + + .. note:: + + A **float** column is the one lossy case, in one direction: pandas + spells missing as ``NaN`` in every float dtype it has (even + ``Float64`` folds ``NaN`` into ``NA``), so a mask-backed float + column, where ``NaN`` is an ordinary value, cannot distinguish its + nulls from its NaNs once converted. :meth:`to_arrow` has a + validity bitmap and is exact for this case. + Returns ------- pandas.DataFrame @@ -9740,13 +9756,124 @@ def to_pandas(self): data = {} for name in self.col_names: col = self[name] - if col.is_ndarray: - data[name] = list(col) - else: - data[name] = col[:] + values = list(col) if col.is_ndarray else col[:] + data[name] = self._pandas_values(pd, col, values) return pd.DataFrame(data) + @staticmethod + def _pandas_scalar_series_values(series, col): + """One pandas Series as something :meth:`extend` can write to *col*. + + The plain ``to_numpy(dtype=...)`` is kept as the fast path, so a + DataFrame that converted before converts identically now. It cannot + represent a missing value, though, and raises on any of pandas' NA + carrying extension dtypes -- which is exactly what :meth:`to_pandas` + emits for a nullable column that holds a null, so without the fallback + below a table would not survive its own round trip. + + The fallback hands the cells over one by one and lets the write path + split nullity out, as it already does for a list of rows containing + ``None``. A sentinel column takes its sentinel there, having nowhere + else to put a null. + + A float ``NaN`` stays a **value** (decision 6), not a null: pandas + spells missing in a float column as NaN and cannot tell the two apart, + so the round trip through a float column is lossy under mask storage + in that one direction. Reading it as a value is what keeps every + NaN-free DataFrame converting exactly as it did before. + """ + kind = np.dtype(col.dtype).kind if col.dtype is not None else "O" + # A float column is exempt from the missing-value check: pandas spells + # missing as NaN there and cannot tell it from a NaN value, and + # decision 6 says value. Everywhere else a missing cell has to leave + # the fast path, which does not merely fail on it -- ``to_numpy`` on a + # text column coerces it to the *string* "None"/"nan". + forces_cells = kind != "f" and bool(series.isna().any()) + if not forces_cells: + try: + return series.to_numpy(dtype=col.dtype) + except (ValueError, TypeError): + pass + + def missing(value): + if is_na_marker(value): + return True + # A float NaN is pandas' missing marker in a text or object column, + # and such a column cannot hold a float as data anyway. Decision + # 6's "NaN is a value" is about *float* columns, which took the + # fast path above and never reach here. + return isinstance(value, float) and value != value + + cells = [None if missing(value) else value for value in series.tolist()] + null_value = getattr(col.spec, "null_value", None) + if null_value is not None: + cells = [null_value if value is None else value for value in cells] + return cells + + @staticmethod + def _pandas_values(pd, col, values): + """*values*, with this column's nulls turned into something pandas reads as NA. + + A null slot holds the fill under mask storage and the sentinel under a + sentinel one, and neither is the row's value, so writing either into a + DataFrame states something the column does not say. Each becomes the + NA of a dtype that has one. + + A column with **no null is returned untouched**, so the dtype a caller + already gets never moves under them. Only a column that actually holds + a null can change, and only to a dtype able to express it -- the same + data-dependent widening ``pyarrow.Table.to_pandas`` does, and for the + same reason: NumPy has no missing value for most kinds. + """ + channel = col._nulls + kind = channel.kind + if kind == NULL_MASK: + nulls = channel.null_mask() # one byte per row, off the sidecar + elif kind == NULL_SENTINEL: + nulls = channel.mask_for_values(values) # in band, from what we just read + else: + # NULL_NONE has no nulls at all, and the dictionary and + # native-None kinds already materialize theirs as None, which is + # exactly what pandas wants. + return values + if not nulls.any(): + return values + + dtype = getattr(values, "dtype", None) + dtype_kind = "O" if dtype is None else dtype.kind + if dtype_kind == "f": + out = np.asarray(values).copy() + out[nulls] = np.nan + return out + if dtype_kind == "M": + out = np.asarray(values).copy() + # With the column's own unit: a bare np.datetime64("NaT") carries + # the generic unit, which NumPy deprecates. + out[nulls] = np.datetime64("NaT", np.datetime_data(out.dtype)[0]) + return out + if dtype_kind == "b" or getattr(channel.spec, "bool_widened_to_uint8", False): + # A sentinel bool is physically uint8 to make room for its 255, so + # it arrives here as an integer; either way the column is logically + # bool, and pandas' "boolean" is the dtype that admits NA. + out = pd.array(np.asarray(values).astype(np.bool_), dtype="boolean") + out[nulls] = pd.NA + return out + if dtype_kind in "iu": + # pandas' nullable integer dtypes keep the width, where letting the + # column widen to float would silently round a large int64. + prefix = "Int" if dtype_kind == "i" else "UInt" + out = pd.array(np.asarray(values), dtype=f"{prefix}{dtype.itemsize * 8}") + out[nulls] = pd.NA + return out + # Text, bytes, complex and ndarray cells have no NA-capable NumPy + # dtype, so they go to object with None -- which is what the dictionary + # and variable-length kinds already produce. + out = list(values) + for i in np.flatnonzero(nulls): + out[i] = None + return out + @classmethod def from_pandas(cls, df, row_cls) -> CTable: # noqa: C901 """Build a :class:`CTable` from a pandas DataFrame. @@ -9895,7 +10022,7 @@ def normalize_pandas_missing(value): ): raw_columns[col.name] = [normalize_pandas_missing(value) for value in series.tolist()] else: - raw_columns[col.name] = series.to_numpy(dtype=col.dtype) + raw_columns[col.name] = cls._pandas_scalar_series_values(series, col) obj.extend(raw_columns, validate=True) return obj diff --git a/tests/ctable/test_null_storage_equivalence.py b/tests/ctable/test_null_storage_equivalence.py index 3bf1e6906..508a6ba61 100644 --- a/tests/ctable/test_null_storage_equivalence.py +++ b/tests/ctable/test_null_storage_equivalence.py @@ -33,9 +33,12 @@ * **complex is mask-only**: no complex value is safe to reserve, so there is no sentinel column to compare against. -One more difference is **not** deliberate -- ``to_pandas`` emits whatever -stands in for a null instead of NA -- and is pinned below as a strict xfail, so -that fixing it trips this suite and the marker comes off. +Everything else agrees, and there are no known unintentional divergences left: +the ``isin`` and ``to_pandas`` leaks this suite was first written to pin are +both fixed, and their tests are now plain assertions. The one remaining +inexactness is not a divergence but a limit of the destination format -- +pandas has no float dtype separating NaN from missing, so a mask float column +cannot round-trip through it (:func:`test_a_mask_float_cannot_round_trip_through_pandas`). """ from __future__ import annotations @@ -91,30 +94,6 @@ def secs(n: int) -> np.timedelta64: #: The row that is null in every table this module builds. NULL_ROW = 1 -#: Kinds whose *fill* and *sentinel* happen to be the same value, so an API -#: that leaks what stands in for a null leaks something indistinguishable -#: either way. A float column fills with NaN and reserves NaN; a timestamp -#: fills with ``int64.min`` and reserves ``int64.min``. They are the kinds -#: where the bug below is invisible -- not the kinds where it is fixed -- so -#: they still assert the correct behaviour, just without the xfail. -INDISTINGUISHABLE_FILL = ("float32", "float64", "timestamp") - - -def leaks_its_fill(kind: str) -> bool: - """Whether a raw read of *kind*'s null slot exposes an ordinary-looking value.""" - return kind not in INDISTINGUISHABLE_FILL - - -def kinds_xfailing_on_leak(reason: str): - """``ALL_KINDS``, with the fill-leaking ones marked ``xfail(strict=True)``.""" - return [ - pytest.param( - kind, - marks=pytest.mark.xfail(strict=True, reason=reason) if leaks_its_fill(kind) else (), - ) - for kind in ALL_KINDS - ] - def annotation_for(spec): if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)): @@ -550,26 +529,105 @@ def test_isin_none_beside_a_real_value(kind): assert s["a"].isin([present, None]).tolist() == want -# --------------------------------------------------------------------------- -# Divergences that are bugs -# --------------------------------------------------------------------------- - - -TO_PANDAS_LEAK = ( - "to_pandas() writes the raw values, so a null arrives as the fill under mask storage " - "and as the sentinel under a sentinel one. Neither is NA, and to_arrow() on the same " - "data is already exact." -) - - -@pytest.mark.parametrize("kind", kinds_xfailing_on_leak(TO_PANDAS_LEAK)) +@pytest.mark.parametrize("kind", ALL_KINDS) def test_to_pandas_agrees(kind): - pd = pytest.importorskip("pandas") + """A null reaches pandas as NA, not as whatever stands in for it.""" + pytest.importorskip("pandas") m, s = both(kind) got, want = m.to_pandas()["a"], s.to_pandas()["a"] assert_same(got.isna().tolist(), want.isna().tolist(), "to_pandas isna") + assert_same(got.isna().tolist(), m["a"].is_null().tolist(), "to_pandas isna vs is_null") assert got.isna()[NULL_ROW], "the null row should read as NA" +@pytest.mark.parametrize("kind", ALL_KINDS) +def test_to_pandas_keeps_the_values_it_does_have(kind): + """Expressing the nulls must not disturb the rows that are not null.""" + pytest.importorskip("pandas") + for t in both(kind): + series = t.to_pandas()["a"] + for i, value in enumerate(logical(t["a"])): + if value is None: + continue + got = series[i] + got = got.item() if hasattr(got, "item") else got + assert got == value, f"row {i}: {got!r} != {value!r}" + + if __name__ == "__main__": pytest.main(["-v", __file__]) + + +# --------------------------------------------------------------------------- +# The pandas round trip +# --------------------------------------------------------------------------- + + +def row_cls_for(kind: str, storage: str): + """A one-column dataclass matching what :func:`build` produces.""" + factory, _values, sentinel = KINDS[kind] + spec = ( + factory(nullable=True, null_storage="mask") + if storage == "mask" + else factory(nullable=True, null_value=sentinel) + ) + return dataclasses.make_dataclass("RoundTripRow", [("a", annotation_for(spec), blosc2.field(spec))]) + + +#: Float is excluded: pandas spells missing as NaN in every float dtype it +#: has, and a mask float column keeps NaN a value (decision 6), so the two +#: cannot survive the trip. :func:`test_a_mask_float_cannot_round_trip_through_pandas` +#: pins that limitation rather than leaving it implicit. +ROUND_TRIP_KINDS = [k for k in ALL_KINDS if not k.startswith("float")] + + +@pytest.mark.parametrize("kind", ROUND_TRIP_KINDS) +@pytest.mark.parametrize("storage", ["mask", "sentinel"]) +def test_pandas_round_trip_keeps_the_nulls(kind, storage): + """to_pandas must emit something from_pandas can read back unchanged.""" + pytest.importorskip("pandas") + t = build(kind, storage) + back = blosc2.CTable.from_pandas(t.to_pandas()[["a"]], row_cls_for(kind, storage)) + assert_same(logical(back["a"]), logical(t["a"]), f"{storage} round trip") + assert logical(back["a"])[NULL_ROW] is None + + +@pytest.mark.parametrize("storage", ["mask", "sentinel"]) +def test_pandas_round_trip_of_a_null_free_column(storage): + """The common case must not be disturbed by any of the above.""" + pytest.importorskip("pandas") + spec = ( + blosc2.int64(nullable=True, null_storage="mask") + if storage == "mask" + else blosc2.int64(nullable=True, null_value=-9) + ) + row_cls = dataclasses.make_dataclass("PlainRow", [("a", int, blosc2.field(spec))]) + t = blosc2.CTable(row_cls, expected_size=16) + t.extend([(1,), (2,), (3,)]) + df = t.to_pandas() + # No null, so nothing widens: the dtype is what it has always been here. + assert df["a"].dtype == np.dtype(np.int64) + back = blosc2.CTable.from_pandas(df, row_cls) + assert logical(back["a"]) == [1, 2, 3] + + +@pytest.mark.parametrize("kind", ["float32", "float64"]) +def test_a_mask_float_cannot_round_trip_through_pandas(kind): + """The one stated limitation, pinned so it stays a known quantity. + + pandas has no float dtype that distinguishes NaN from missing -- even + ``Float64`` folds a NaN into ``NA`` -- so a mask float column's null comes + back as a NaN *value*. A sentinel float column is lossless precisely + because NaN is what it means by null in the first place. + """ + pytest.importorskip("pandas") + mask_back = blosc2.CTable.from_pandas(build(kind, "mask").to_pandas()[["a"]], row_cls_for(kind, "mask")) + assert mask_back["a"].null_count() == 0, "the null came back as a NaN value" + assert np.isnan(mask_back["a"][:][NULL_ROW]) + + sentinel_back = blosc2.CTable.from_pandas( + build(kind, "sentinel").to_pandas()[["a"]], row_cls_for(kind, "sentinel") + ) + assert sentinel_back["a"].null_count() == 1 + # to_arrow is the export that keeps the distinction for both storages. + assert build(kind, "mask").to_arrow()["a"].null_count == 1 From 71db3057461ef82243eee15cbb085a7444249540 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 11:19:08 +0200 Subject: [PATCH 23/47] Let cov() see a mask column's nulls cov() already dropped nulls listwise, and already meant to: the comment at the site says so. It just computed nullity with _null_mask_for, the in-band sentinel test, which a mask column has no sentinel to answer -- so every null read as an ordinary value and the fill was averaged into the result. On [(1,2),(null,null),(3,6)] that gave 2.33 where a table holding the same rows without the null gives 2.0. Nullity now comes from the null channel: the sidecar for a mask column, and the same in-band test as before for a sentinel one, answered from the values already read rather than by reading the column again. The dictionary and variable-length kinds cannot reach it, having been rejected on dtype further up. Listwise is kept, not changed to the pairwise dropping pandas does: it was the existing intent, and it keeps every entry of the matrix computed over one set of rows, so the result stays consistent with itself. That was documented only in a comment, and is now in the docstring where a caller will find it, along with the pandas difference. A mask float column's NaN is still data (decision 6), so it propagates into the result rather than dropping the row -- the same answer sum() already gives, and asserted rather than left to be discovered. Added to the equivalence oracle, which checks both storages against each other and against a table holding the same rows with the null one simply absent. Non-vacuous by mutation: restoring the old blind spot fails 7 of the 8 new cases. tests/ctable: 2550 passed, 1 xfailed. Co-Authored-By: Claude Opus 5 --- src/blosc2/ctable.py | 23 ++++++- tests/ctable/test_null_storage_equivalence.py | 67 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index e1a2579fa..db347234b 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -7458,6 +7458,13 @@ def cov(self) -> np.ndarray: cast to int (0/1) before computation. Complex columns raise :exc:`TypeError`. + Nulls are dropped **listwise**: a row that is null in any column is + excluded from every column, so all entries of the result are computed + over the same rows and the matrix stays consistent with itself. This + differs from ``pandas.DataFrame.cov()``, which drops pairwise. Rows + are dropped by what :meth:`Column.is_null` reports, never by the value + standing in for a null, so both null storages give the same answer. + Returns ------- numpy.ndarray @@ -7469,7 +7476,8 @@ def cov(self) -> np.ndarray: TypeError If any column has an unsupported dtype (complex, string, …). ValueError - If the table has fewer than 2 live rows (covariance undefined). + If the table has fewer than 2 live rows (covariance undefined), or + fewer than 2 rows survive the listwise null drop. """ for name in self.col_names: col_info = self._schema.columns_by_name.get(name) @@ -7498,7 +7506,18 @@ def cov(self) -> np.ndarray: for name in self.col_names: col = self[name] arr = col[:] - nm = col._null_mask_for(arr) + channel = col._nulls + if channel.kind == NULL_MASK: + # Off the sidecar. The in-band test below cannot see these: + # a mask column has no sentinel, so it would call every null + # row an ordinary value and average in the fill. + nm = channel.null_mask() + else: + # In band, answered from the values already read rather than + # by reading the column a second time. All-False for a column + # with no nulls at all; the dictionary and variable-length + # kinds cannot reach here, having been rejected on dtype. + nm = channel.mask_for_values(arr) if nm.any(): null_union = nm if null_union is None else (null_union | nm) raw_arrays.append(arr) diff --git a/tests/ctable/test_null_storage_equivalence.py b/tests/ctable/test_null_storage_equivalence.py index 508a6ba61..61518e984 100644 --- a/tests/ctable/test_null_storage_equivalence.py +++ b/tests/ctable/test_null_storage_equivalence.py @@ -289,6 +289,73 @@ def test_ordered_non_numeric_reductions_agree(kind, op): assert_same(getattr(m["a"], op)(), getattr(s["a"], op)(), op) +@pytest.mark.parametrize("kind", [*NUMERIC_KINDS, "bool"]) +def test_cov_agrees(kind): + """cov() drops a null row listwise, and must learn which rows those are. + + It asked the values, which for a mask column means asking a fill that + looks like ordinary data, so the null was averaged in. The reference is a + table holding the same rows with the null one simply absent. + """ + m, s = both(kind) + factory, values, _sentinel = KINDS[kind] + kept = [(v, i) for i, v in enumerate(values) if v is not None] + row_cls = dataclasses.make_dataclass( + "CovRef", + [ + ("a", annotation_for(factory()), blosc2.field(factory())), + ("n", int, blosc2.field(blosc2.int64())), + ], + ) + reference = blosc2.CTable(row_cls, expected_size=16) + reference.extend(kept) + + # Column "g" is text, which cov() rejects, so pair "a" with a numeric one. + got = np.asarray(m.select(["a"]).cov()) + want = np.asarray(s.select(["a"]).cov()) + np.testing.assert_allclose(got, want, err_msg="mask and sentinel disagree") + np.testing.assert_allclose( + got, np.asarray(reference.select(["a"]).cov()), err_msg="neither matches a null-free table" + ) + + +def test_cov_drops_a_null_row_from_every_column(): + """Listwise: a null in one column removes the row from the others too.""" + row_cls = dataclasses.make_dataclass( + "CovRow", + [ + ("a", int, blosc2.field(blosc2.int64(nullable=True, null_storage="mask"))), + ("b", int, blosc2.field(blosc2.int64())), + ], + ) + t = blosc2.CTable(row_cls, expected_size=16) + t.extend([(1, 2), (None, 7), (3, 6), (5, 1)]) + + plain_cls = dataclasses.make_dataclass( + "PlainCovRow", + [("a", int, blosc2.field(blosc2.int64())), ("b", int, blosc2.field(blosc2.int64()))], + ) + reference = blosc2.CTable(plain_cls, expected_size=16) + reference.extend([(1, 2), (3, 6), (5, 1)]) + + np.testing.assert_allclose(np.asarray(t.cov()), np.asarray(reference.cov())) + + +def test_cov_keeps_a_mask_floats_nan_as_data(): + """Decision 6 again: only mask=False is dropped, so a NaN poisons the result.""" + row_cls = dataclasses.make_dataclass( + "NanCovRow", + [ + ("a", float, blosc2.field(blosc2.float64(nullable=True, null_storage="mask"))), + ("b", float, blosc2.field(blosc2.float64())), + ], + ) + t = blosc2.CTable(row_cls, expected_size=16) + t.extend([(1.0, 2.0), (float("nan"), 7.0), (3.0, 6.0)]) + assert t["a"].null_count() == 0 + assert np.isnan(np.asarray(t.cov())[0, 0]) + + # --------------------------------------------------------------------------- # Grouping # --------------------------------------------------------------------------- From cb651716c1071ca2fcd2f45bef0ba74de488c519 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 9 Aug 2026 11:45:09 +0200 Subject: [PATCH 24/47] Skip the Arrow equivalence cases when pyarrow is absent test_to_arrow_agrees called to_arrow() with no importorskip, so a CI job without pyarrow got ten ImportErrors instead of ten skips. The pandas cases in the same file were guarded; the Arrow one was simply missed. Also guards the to_arrow assertion at the end of test_a_mask_float_cannot_round_trip_through_pandas, which is a pandas test that reaches for Arrow to show what pandas loses. It only escaped the same failure because that job has no pandas either, so it skipped before getting there -- a job with pandas and without pyarrow would have failed it. Verified by making the two modules unimportable: the whole ctable suite is 2191 passed, 103 skipped, 0 failed without either of them, and unchanged at 2550 passed with both. Co-Authored-By: Claude Opus 5 --- tests/ctable/test_null_storage_equivalence.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ctable/test_null_storage_equivalence.py b/tests/ctable/test_null_storage_equivalence.py index 61518e984..9f78c56b0 100644 --- a/tests/ctable/test_null_storage_equivalence.py +++ b/tests/ctable/test_null_storage_equivalence.py @@ -499,6 +499,7 @@ def test_a_null_never_satisfies_a_predicate(kind): @pytest.mark.parametrize("kind", ALL_KINDS) def test_to_arrow_agrees(kind): """Arrow has a validity bitmap of its own, so this is the exact comparison.""" + pytest.importorskip("pyarrow") m, s = both(kind) got, want = m.to_arrow()["a"], s.to_arrow()["a"] assert_same(got.null_count, want.null_count, "to_arrow null_count") @@ -697,4 +698,5 @@ def test_a_mask_float_cannot_round_trip_through_pandas(kind): ) assert sentinel_back["a"].null_count() == 1 # to_arrow is the export that keeps the distinction for both storages. + pytest.importorskip("pyarrow") assert build(kind, "mask").to_arrow()["a"].null_count == 1 From 4dbeedeb40ae32dc401e600fa3df8e02621593f0 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 09:07:21 +0200 Subject: [PATCH 25/47] Give a null predicate its third value A comparison against a null was collapsed to False at the leaf. That is right for WHERE, which keeps what a predicate is true for, and it is why every consumer could stay two-valued through ten phases of mask-storage work. It is wrong for anything that reads the result as a value rather than as a filter: ~(t.a > 10) inverted the collapsed False and returned every null row, the precise opposite of the intent, pinned since Phase 1 as a strict xfail. Comparisons now return a NullableBoolExpr carrying two disjoint channels -- definitely-true, and unknown -- and &, |, ^ and ~ combine them by Kleene's rules. The truth channel stays collapsed, so where() returns exactly the rows it did before and every existing consumer is untouched; only the four boolean operators consult the second channel. The type subclasses LazyExpr rather than wrapping it. A wrapper fails on isinstance, which where(), _try_index_where and user-shaped code all use, and it cannot get the reflected operators right: Python offers __rand__ to a subclass first, so plain_expr & nullable_predicate reaches the Kleene operator instead of silently collapsing. Operand.__and__/__or__/ __xor__ defer explicitly for an NDArray on the left, which is not a superclass of anything here. The string form is made exact too, not left conservative. Pushing validity to the negation point drops rows SQL keeps -- NULL AND FALSE is FALSE, so its negation is true -- and mask-based-nulls.md accepted that while the operator form was also wrong, in the other direction. With the operator form exact, a conservative string form would mean the two spellings of one predicate returning different rows, which is worse than either error alone. The rewriter carries both channels under a negation and emits byte-identical text everywhere else, so the index planner sees the shapes it already knows. Two more producers had to be found. A nullable bool column is itself a three-valued predicate, with its third value in band for sentinel storage and in the sidecar for mask storage. And a dictionary != returned its nulls: the reserved code differs from every value's code, so codes != target was true for a null row. == None stays two-valued, being the is-null test that Column.is_null is built on. _expression_has_or becomes _expression_defeats_global_null_filter and answers True for a negation as well: a global null post-filter is wrong for both shapes, for the same reason. Negations do not reach an exact plan today, so this is the same belt-and-braces the OR bail already was. Verified by tests/ctable/test_kleene_logic.py (76 tests): all nine cells of each binary operator over both storages and both kinds of operand; De Morgan and double negation hold and the excluded middle deliberately does not; and a NumPy Kleene oracle over 8 expression shapes x {mask, sentinel} x {scan, indexed} checking both query forms and the unknown channel. Suite is 9217 passed, 29 skipped. Measured on 20M rows: every predicate shape unchanged except a negated two-column conjunction, 106 -> 122 ms (1.15x), which is what the exact answer costs. Co-Authored-By: Claude Opus 5 --- RELEASE_NOTES.md | 43 ++ doc/reference/ctable.rst | 35 ++ plans/kleene-logic.md | 193 ++++++++ plans/mask-based-nulls.md | 23 +- src/blosc2/ctable.py | 456 +++++++++++++++-- src/blosc2/ctable_indexing.py | 45 +- src/blosc2/ctable_nulls.py | 200 ++++++-- src/blosc2/ndarray.py | 20 + tests/ctable/test_kleene_logic.py | 520 ++++++++++++++++++++ tests/ctable/test_null_aware_indexes.py | 22 +- tests/ctable/test_null_expressions.py | 25 +- tests/ctable/test_null_predicate_rewrite.py | 45 +- 12 files changed, 1497 insertions(+), 130 deletions(-) create mode 100644 plans/kleene-logic.md create mode 100644 tests/ctable/test_kleene_logic.py diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e4052b283..df0f7597b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -100,6 +100,49 @@ costs one decompression pass (33 ms for a 20M-row `int64` column) because the incremental per-block summaries folded during writes carry no validity; a nullable column with no nulls keeps that fast path untouched. +#### Predicates over nulls follow three-valued (Kleene) logic + +A comparison against a null is neither true nor false. It is now **unknown**, +the third value SQL and Arrow both use, and `&`, `|`, `^` and `~` combine it by +Kleene's rules instead of collapsing it to `False` at the leaf: + +```python +t.where(t.price > 10) # rows definitely above 10 +t.where(~(t.price > 10)) # rows definitely *not* above 10 — nulls in neither +``` + +`where()` keeps what a predicate is **true** for, so the rows it returns for a +plain comparison are unchanged. What this fixes is everything built on top of +one. `~(t.price > 10)` used to invert a null that had already been collapsed to +`False` and so returned every null row — the exact opposite of the intent — and +`~((a > 10) & (b == 999))` dropped rows that qualify, because `unknown & false` +is *false*, not unknown, and only a real third value can express that. Both +query forms are covered, and both now agree with SQL: the string form carries +the second channel through an AST rewrite under negation. + +A predicate can be asked about its unknown rows rather than only filtered with: + +```python +p = t.price > 10 +p.is_null() # boolean array: the rows it cannot answer for +p.null_count() +t.where(p.fillna(True)) # keep what cannot be ruled out +``` + +`fillna(False)` is the other reading, and is what `where()` applies implicitly. + +Predicates over non-nullable columns are untouched and cost nothing extra; the +result of a nullable comparison is still a `blosc2.LazyExpr`, so it computes, +indexes and plans exactly as before. Measured cost of the exact answer: a +negated two-column conjunction over a 20M-row nullable table runs 1.15x slower +than the wrong answer it replaces; every other predicate shape is unchanged. + +Two consequences worth knowing. `t.where(dict_col != "x")` no longer returns the +rows where `dict_col` is null (its reserved code differs from every value's, so +it used to match); `dict_col == None` remains how to ask for them. And +`Column.isin()` stays deliberately two-valued — it returns a materialized array +and has its own spelling for nulls (`None` among the values). + ### Bug fixes - **`group_by` returned the wrong `min` for a `bool` value column.** The diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index 6cb4b2c54..6255ded8f 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -183,6 +183,41 @@ Indexes understand both storages: a column index summarises only the rows that carry a value, so ``min``/``max`` and ``where`` are as fast on a nullable column as on a plain one. See `Indexes`_. +.. _kleene-logic: + +Predicates over nulls: three-valued logic +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A comparison against a null is neither true nor false. SQL and Arrow call +that third value **unknown**, and Blosc2 follows the same (Kleene) rules, for +both query forms:: + + t.where(t.price > 10) # rows where the price is definitely above 10 + t.where("price > 10") # the same rows + t.where(~(t.price > 10)) # rows definitely *not* above 10 — nulls excluded + +A null row satisfies neither a predicate nor its negation: ``where()`` keeps +what is **true**, so the unknown rows are dropped exactly like the false ones. +That is what makes ``~`` behave — a naive bitwise negation of "null compares +False" would *select* every null. + +``&`` and ``|`` follow the same logic, which is not simply "drop any row a null +touched":: + + (t.price > 10) & (t.qty == 0) # false where qty != 0, even if price is null + (t.price > 10) | (t.qty == 0) # true where qty == 0, even if price is null + +The unknown rows are not lost, and a predicate can be asked about them:: + + p = t.price > 10 + p.is_null() # boolean array: rows the predicate cannot answer for + p.null_count() # how many + t.where(p.fillna(True)) # keep the rows that cannot be ruled out + +A predicate over a non-nullable column has no unknown rows and costs nothing +extra; ``isin()`` is deliberately two-valued and has its own spelling for +nulls (put ``None`` in the values). + Converting between them ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/plans/kleene-logic.md b/plans/kleene-logic.md new file mode 100644 index 000000000..20c644341 --- /dev/null +++ b/plans/kleene-logic.md @@ -0,0 +1,193 @@ +# Three-valued (Kleene) logic for CTable predicates + +> **Status: complete — landed 2026-08-10.** Both query forms reason in three values, +> the `strict=True` xfail this work inherited is now a passing test, and the whole suite +> (9217 tests) is green. Three premises from the design below were disproven while building it +> and are corrected in place, each beside the text it corrects: the result type must **subclass** +> `LazyExpr` rather than wrap it (§The type), the dictionary `!=` bug was not in scope and turned +> out to be the same bug (§Producers), and the string form's "conservative is good enough" ruling +> from `plans/mask-based-nulls.md` does not survive contact with an exact operator form +> (§The string form). Drafted and implemented 2026-08-10. +> +> This discharges **decision 8** of `plans/mask-based-nulls.md` ("Kleene three-valued logic stays +> out of scope … named deferred follow-up") and the follow-up entry that inherited the +> operator-form negation leak. Read that document for how nulls are *stored*; read this one for +> what a predicate *means* when it meets one. + +## Context + +Every phase of the mask-nulls work collapsed a null comparison to `False` at the leaf — +`_null_aware_compare` returns `raw & ~null_pred`, `rewrite_null_predicates` conjoins +`& (a != 999)` onto each leaf. That is exactly right for `WHERE`, which keeps what a predicate +is *true* for, and it is why every consumer could stay two-valued for ten phases. + +It is wrong for anything that reads the result as a boolean *value* rather than as a filter: + +- **`~` inverts the collapse.** `~(t.a > 10)` returned every null row — the precise opposite of + the intent. Pinned since Phase 1 as a `strict=True` xfail + (`test_operator_form_negation_drops_nulls`). +- **The string form's fix for the same problem is not exact.** Pushing validity to the negation + point — `(~(a > 10)) & valid_a` — drops rows SQL keeps: `not (a > 10 and b == 999)` with a false + second term is `not FALSE` = true, because `NULL AND FALSE` collapses to `FALSE`. Pinned as + `test_negation_over_and_corner_is_conservative`, deliberately, with the note that the two forms + disagreed here. +- **A dictionary `!=` returned its nulls.** The reserved code differs from every value's code, so + `codes != target` was true for a null row. Same bug, different subsystem — found while auditing + the producers, not anticipated. + +The missing piece is not more guards. It is the third value itself: `unknown & false` is **false**, +not unknown, and no conjunction of validity predicates can say that. That is Kleene's logic, and it +needs the null channel to survive as far as the operator that consumes it. + +## Decisions + +1. **The truth channel stays collapsed.** A three-valued predicate carries `(true, unknown)` where + `true` is already "definitely true" (disjoint from `unknown`), not a raw comparison plus a flag. + Every existing consumer reads the truth channel and gets exactly today's answer; only the four + boolean operators consult the second one. This is what keeps the diff at the edges of the system + rather than through it. +2. **`WHERE` semantics do not change.** SQL keeps what is true, so an unknown row is dropped like a + false one. The rows returned for a plain comparison are byte-for-byte what they were. +3. **Both query forms, one semantics.** The string form is rewritten to the same rules rather than + left conservative — see §The string form for why "it only ever drops rows" stopped being an + acceptable divergence once the operator form was exact. +4. **`isin()` stays two-valued.** It returns a materialized array, is documented as such, and has + its own spelling for nulls (`None` among the values). Making it three-valued would change a + public return type to fix a question it already answers. +5. **Nothing is paid by a predicate that meets no null.** A comparison over a non-nullable column + returns a plain `LazyExpr`, as before; the machinery starts only where `_combined_null_pred` + already returned something. + +## The type + +`NullableBoolExpr` (`ctable.py`, beside `NullableExpr`) holds `_true`, `_null` and its table. + +> **Correction (2026-08-10): it must *subclass* `LazyExpr`, not wrap it.** The first +> implementation was a wrapper with `__getattr__` forwarding, in the shape `NullableExpr` already +> uses. It fails on `isinstance`: `CTable.where` type-checks its argument, `_try_index_where` wants +> `.expression`/`.operands`, and a test helper in `test_dictionary_column.py` — plain user-shaped +> code — does `mask.compute() if isinstance(mask, blosc2.LazyExpr) else mask` and silently took +> the wrong branch. Subclassing fixes all of them at once, and buys two things the wrapper could +> not: +> +> - **Reflected operators come for free.** Python offers `__rand__`/`__ror__` to the right operand +> first when its type is a subclass of the left's, so `plain_expr & three_valued` reaches the +> Kleene operator instead of collapsing. The wrapper needed `blosc2.Operand` to defer explicitly. +> - **No forwarding surface to keep in sync.** An instance *adopts* the truth channel's state +> (`self.__dict__.update(true_expr.__dict__)`), so it does not merely behave like the collapsed +> predicate — it is it. +> +> `blosc2.Operand.__and__`/`__or__`/`__xor__` still defer explicitly (`_defers_boolean_op`, keyed +> on `_kleene_channels`), because an `NDArray` on the left is not a superclass of anything here. +> That also picks up a `Column` on the right, which was already a latent asymmetry. + +The rules, with `None` meaning "this operand is two-valued": + +| op | true | unknown | +|---|---|---| +| `a & b` | `ta & tb` | `(na & (tb\|nb)) \| (nb & (ta\|na))` | +| `a \| b` | `ta \| tb` | `(na & ~tb) \| (nb & ~ta)` | +| `a ^ b` | `(ta ^ tb) & ~null` | `na \| nb` | +| `~a` | `~ta & ~na` | `na` | + +Public surface on the result: `is_null()`, `notnull()`, `null_count()`, `fillna(bool)`. +`fillna(False)` is what `where()` does implicitly; `fillna(True)` is "keep what I cannot rule out". + +## Producers + +`Column._null_aware_compare` is the main one and its change is one line — the collapsed predicate +it already built becomes the truth channel. Four more had to be found: + +- **`Column._kleene_channels`** — a nullable *bool column* is itself a three-valued predicate, with + its third value in band for sentinel storage (`255`) and in the sidecar for mask storage. The + boolean operators on `Column` route through it, so `t.flag & t.other` and `~t.flag` are Kleene. + Non-boolean columns are tested for **before** the null channel is asked, so `t.flags & 0x04` + builds no predicate it will discard. +- **`NullableExpr` comparisons** — `(t.a + 1) > 5` carries the arithmetic's NaN-null channel into + the comparison result. `__ne__` already had the collapse; the other five gained it. +- **utf8** — `_utf8_compare_scalar`/`_utf8_compare_column`. These are eager physical-length arrays, + and neither a `StringDType` comparison nor a raw-byte span scan is something the lazy layer + evaluates, so the null channel costs a scan of its own. It is therefore **deferred**: the channel + may be a zero-argument callable, resolved on first use, so an ordinary `==` never pays for it and + only `~`/`is_null()` does. +- **dictionary** — `_dictionary_eq`. + +> **Correction (2026-08-10): the dictionary case is a bug fix, not plumbing.** `!=` compared codes, +> and a null row's reserved code differs from the target's like any other, so `t.where(t.d != "x")` +> **returned every null**. The three-valued result fixes it (`exclude_nulls=` on the wrap, True for +> every negated form and False for a positive one, where no null row can carry the code being +> looked for anyway). `d == None` stays two-valued: it is the is-null test, not a comparison +> against a value, and making it unknown-everywhere would have broken `Column.is_null`, which is +> built on it. + +## The string form + +`_NullPredicateRewriter` grows `_channels(node)`, returning `(true_ast, null_ast)` for a subtree, +and uses it **only under a negation** — so a non-negated expression is rewritten to exactly the +text it was before, and the index planner sees the shapes it already knows. + +> **Correction (2026-08-10) to `plans/mask-based-nulls.md` §Expression layer.** That document +> settled for the conservative negation guard on the grounds that "rows are only ever dropped, +> never wrongly returned". That was a defensible ruling while the operator form was *also* wrong, +> in the other direction. With the operator form exact, leaving the string form conservative would +> mean `t.where("...")` and `t.where(t...)` returning different rows for the same predicate — a +> worse property than either error alone. Both are exact now, and the test that pinned the +> divergence (`test_negation_over_and_corner_is_conservative`) is rewritten in place as +> `test_negation_over_and_corner_agrees_with_sql`, asserting the two forms agree. + +Two details: + +- **A negated *leaf* keeps its short form.** `~(leaf & guard) & guard` is `~leaf & guard`, so the + common shape is special-cased and emits the same text it always did. Only a negated *combination* + grows the second channel. +- **`~~x` is folded.** A leaf's unknown channel is `~guard`, and negating it back is what every + rewritten negation does, so without the fold every output carried a visible double negative. + +`_expression_has_or` — renamed `_expression_defeats_global_null_filter` — now answers True for a +negation too. A *global* null post-filter is wrong for the same reason in both cases: the +predicate's answer for a null row is not simply "no match". The exact-position index paths +therefore fall back to the scan for a negation, and the segment path (which evaluates the +predicate, so it is already exact per leaf) skips the filter. Measured: negations do not reach an +exact plan today at all, so this is the same belt-and-braces the OR bail already was. + +## Verification + +`tests/ctable/test_kleene_logic.py` (76 tests): + +- **The truth tables**, all nine cells of `&`, `|`, `^` and all three of `~`, over both storages + and both ways of producing a three-valued operand (a comparison, and a nullable bool column). + A test reads a row's value the way a user would: true if `where()` keeps it, unknown if + `is_null()` says so. +- **Algebraic identities**: De Morgan holds, double negation is the identity — and the law of the + excluded middle **does not**, pinned so no future simplification of the operators can quietly + restore two-valued logic. +- **A NumPy Kleene oracle** over 8 expression shapes × {mask, sentinel} × {scan, indexed}, checking + the string form, the operator form and the unknown channel against a transcription of the truth + tables. 32 combinations agreeing, in the same shape as the Phase 7 oracle. +- **Every V1 kind** negates the same way (int, float, string, bytes, utf8, both storages), plus the + dictionary `!=` fix and utf8 negation. +- **Operand order**: `plain & three_valued` stays three-valued, including with a materialized + `NDArray` on the left. +- **Consumers**: `t[pred]`, `sum(where=pred)`, `assign(flag=pred)`, and a sorted view (whose + `is_null()` must follow the view's order, not the physical one). + +Changed elsewhere: `test_null_expressions.py`'s `test_inverted_comparison_selects_null_rows` became +`test_inverted_comparison_drops_null_rows` (it documented the old behaviour, in its name), and the +strict xfail became a plain test. + +**Measured** on a 20M-row nullable float column: `where(a > 50)`, `where(~(a > 50))` and both +string equivalents are unchanged (170 ms, scan-bound); `where(~((a > 50) & (b < 20)))` goes +106 ms → 122 ms, **1.15x**, which is the price of the extra channel and buys the answer being +right. + +## Named follow-ups + +- **A mask-storage predicate cannot become a computed column.** `t.assign(flag=t.a > 10)` raises + for a mask column because the predicate reads the `.notnull` sidecar, which is not a stored + column of the table. Pre-existing (the collapsed predicate carried the same operand) and + unrelated to three-valued logic, but it is what stops `assign` from being tested on both + storages. +- **A three-valued column of results.** `fillna`/`where` are the only ways out of the third value + today; storing one would need a computed column with its own validity sidecar. +- **`isin` and the other eager predicates** (`startswith`, `contains`) stay two-valued. If they + ever return a lazy predicate, they should return this one. diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 51ca78ad7..57d927b75 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -93,7 +93,12 @@ answers. 8. **Kleene three-valued logic stays out of scope.** Masks make it possible (they supply the validity channel `plans/enhancing-ctable.md` §Gap C named as the blocker), but `_null_aware_compare` deliberately collapses null → `False` (SQL `WHERE` semantics). - Named deferred follow-up. + Named deferred follow-up. *(Discharged 2026-08-10 — see `plans/kleene-logic.md`. The collapse + stays, as the **truth channel** of a three-valued result; what changed is that the null + channel now survives alongside it, which is what `~` needs. `WHERE` semantics are unchanged. + That work also reverses this document's ruling that a conservative string-form negation is + good enough, and corrects two things recorded below: the negation caveat under §Expression + layer, and the divergence pinned by `test_negation_over_and_corner_is_conservative`.)* 9. **An absent sidecar means all-valid.** The `.notnull` array is materialized lazily, on the first write that actually contains a null; a mask-storage column with no `.notnull` key on disk is a valid — and expected common — state meaning "no nulls so far". Null-free nullable @@ -698,6 +703,14 @@ Phase 1**, landing right after the `NullChannel` refactor and before any mask wo > `NullableExpr` with `__invert__` — and folds naturally into decision 8's deferred Kleene > follow-up rather than Phase 1. > +> **Resolved 2026-08-10 (`plans/kleene-logic.md`).** The predicted fix is what shipped, with one +> departure: the boolean analogue **subclasses** `LazyExpr` instead of wrapping it, because +> `isinstance` checks at three consumers (and in user-shaped test code) took the wrong branch for a +> wrapper. The xfail is now a plain test. The "intentional divergence" pinned beside it did **not** +> survive: with the operator form exact, a conservative string form would have meant the two +> spellings of one predicate returning different rows, which is worse than either error alone — +> so the string rewrite carries two channels under a negation and both forms are now SQL-exact. +> > **Addendum 2 (2026-08-08): Phase 1 only did the sentinel half, and Phase 6 shipped the gap.** > `_rewrite_null_predicates` tested `kind_of_spec(spec) != NULL_SENTINEL` and skipped everything > else, so a mask column got no guard at all — and the same one-kind test in @@ -1006,8 +1019,12 @@ channel to one more builder rather than to teach a new subsystem about nulls. - **A mask *key* column back on the dense single-key path.** It leaves because `_CodedKeyChunk.codes` are chunk-local, not the dense global ints that path indexes with. - **`__setitem__`'s fast paths** for mask columns (deferred in Phase 4, still unmeasured). -- **Kleene three-valued logic**, decision 8 — which now also owns the operator-form negation leak - pinned as a `strict=True` xfail in `tests/ctable/test_null_predicate_rewrite.py`. +- ~~**Kleene three-valued logic**, decision 8 — which now also owns the operator-form negation leak + pinned as a `strict=True` xfail in `tests/ctable/test_null_predicate_rewrite.py`.~~ + **Done 2026-08-10**, `plans/kleene-logic.md`: the xfail is a passing test, the string form's + conservative negation became exact, and a dictionary `!=` stopped returning its nulls. Cost of + the exact answer, measured: 1.15x on a negated two-column conjunction over 20M rows; every other + predicate shape unchanged. - **Validity through `_ColumnSummaryAccumulator`** (Phase 10). The per-block summaries folded during writes carry no validity, so a nullable column holding a null cannot use them and pays a decompression pass at `close()` — 1.8 ms → 33.2 ms for a 20M-row `int64`. `extend`'s feed site diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index db347234b..915a24578 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -1092,34 +1092,51 @@ def __neg__(self): # ---- comparisons: IEEE NaN already gives SQL False, except for != ---- + def _compare(self, other, raw_result): + """Three-valued comparison result, carrying the nulls of both sides. + + The truth channel is the collapsed predicate — IEEE already gives False + for every ordered comparison against a NaN null, and ``~pred`` covers + ``!=``, which IEEE says is True — and the null channel is what makes + ``~`` able to tell a null row from a false one. + """ + pred = self._null_pred + other_pred = None + if isinstance(other, Column): + other_pred = other._raw_null_pred() + elif isinstance(other, NullableExpr): + other_pred = other._null_pred + if other_pred is not None: + pred = pred | other_pred + return NullableBoolExpr(raw_result & ~pred, self._table, pred) + def __lt__(self, other): - return NotImplemented if self._defer(other) else self._expr < self._operand(other) + if self._defer(other): + return NotImplemented + return self._compare(other, self._expr < self._operand(other)) def __le__(self, other): - return NotImplemented if self._defer(other) else self._expr <= self._operand(other) + if self._defer(other): + return NotImplemented + return self._compare(other, self._expr <= self._operand(other)) def __gt__(self, other): - return NotImplemented if self._defer(other) else self._expr > self._operand(other) + if self._defer(other): + return NotImplemented + return self._compare(other, self._expr > self._operand(other)) def __ge__(self, other): - return NotImplemented if self._defer(other) else self._expr >= self._operand(other) + if self._defer(other): + return NotImplemented + return self._compare(other, self._expr >= self._operand(other)) def __eq__(self, other): - return NotImplemented if self._defer(other) else self._expr == self._operand(other) + if self._defer(other): + return NotImplemented + return self._compare(other, self._expr == self._operand(other)) def __ne__(self, other): - # IEEE says nan != x is True; SQL null semantics say a null satisfies - # no comparison. Guard both sides (Column and NullableExpr operands - # carry their own null predicates). - result = (self._expr != Column._unwrap_operand(other)) & ~self._null_pred - other_pred = None - if isinstance(other, Column): - other_pred = other._raw_null_pred() - elif isinstance(other, NullableExpr): - other_pred = other._null_pred - if other_pred is not None: - result = result & ~other_pred - return result + return self._compare(other, self._expr != Column._unwrap_operand(other)) # ---- reductions: skip nulls and dead physical rows ---- @@ -1130,6 +1147,7 @@ def _reduction_mask(self, where=None): t = self._table n_rows = t._known_n_rows() mask = None if (n_rows is not None and n_rows == len(t._valid_rows)) else t._valid_rows + where = _sql_predicate(where) if where is not None: mask = where if mask is None else mask & where nonnull = ~self._null_pred @@ -1169,6 +1187,204 @@ def max(self, *, where=None): return self._minmax("max", where) +class NullableBoolExpr(blosc2.LazyExpr): + """A predicate whose value may be *unknown* — Kleene three-valued logic. + + Comparing against a null yields neither true nor false. SQL and Arrow + both call that third value **unknown**, and Kleene's logic is what ``&``, + ``|`` and ``~`` do once it exists. This class is that value made + representable: two disjoint boolean channels over the same raw physical + arrays the predicate was built from. + + * :attr:`_true` — True exactly where the predicate is **definitely true**; + * :attr:`_null` — True where it is **unknown**, a null having taken part. + + Rows that are definitely false are simply in neither channel. + + Keeping the truth channel *already collapsed* — rather than storing a raw + comparison beside a null flag — is what lets every existing consumer stay + untouched. ``WHERE`` keeps the rows a predicate is true for, so anything + reading only ``_true`` gets exactly the answer it got before three-valued + logic existed. What changes is ``~``: two-valued negation turns *unknown* + into a match, which is the bug this class fixes at the source rather than + at each consumer. + + ========== ========== ============ =========== ========== + ``a`` ``b`` ``a & b`` ``a | b`` ``~a`` + ========== ========== ============ =========== ========== + true unknown unknown true false + false unknown **false** unknown true + unknown unknown unknown unknown unknown + ========== ========== ============ =========== ========== + + The second row is the whole reason three values need their own algebra: + ``false & unknown`` is *false*, not unknown, because no value of the + missing operand could make the conjunction true. A design that dropped + every row touched by a null would get that one wrong. + + An operand with no null channel (a plain :class:`blosc2.LazyExpr`, a + non-nullable column) is two-valued and combines as such — the tables above + with the *unknown* column removed — so a predicate that never meets a null + costs nothing. + + The two channels live in **physical** coordinates, like the lazy + predicates they are built from; :meth:`is_null` converts to logical rows. + + **It is a** :class:`blosc2.LazyExpr` — subclassing rather than wrapping. + The truth channel is not something this object *holds*, it is what this + object *is*: an instance adopts the state of the collapsed predicate, so + every consumer that computes it, indexes with it, plans an index around it + or type-checks it goes on working with no idea a third value exists. Only + the four boolean operators are overridden, and Python's subclass rule for + reflected operators means ``plain_expr & three_valued`` reaches them too. + """ + + #: Class-level defaults, so an instance that arrives without them (a copy, + #: an unpickle) degrades to the two-valued predicate it already is rather + #: than raising deep inside an operator. + _table = None + _null_src = None + + def __init__(self, true_expr, table, null_pred): + if not isinstance(true_expr, blosc2.LazyExpr): + true_expr = blosc2.lazyexpr(true_expr) + self.__dict__.update(true_expr.__dict__) + self._true = true_expr + self._table = table + # May be a zero-argument callable, resolved on first use: for utf8 and + # dictionary columns building the null channel costs a scan of its own, + # and only ``~``/:meth:`is_null` ever needs it. + self._null_src = null_pred + + @property + def _null(self): + """The unknown channel, materializing a deferred one on first use.""" + src = self._null_src + if callable(src): + src = src() + self._null_src = src + return src + + def __repr__(self): + return f"NullableBoolExpr({self.expression!r})" + + # ------------------------------------------------------------------ + # Kleene algebra + # ------------------------------------------------------------------ + + @staticmethod + def _channels(operand): + """``(definitely-true, unknown)`` for any boolean operand. + + *unknown* is ``None`` for a two-valued operand, which every rule below + reads as "no row is unknown". + """ + if isinstance(operand, (NullableBoolExpr, Column)): + return operand._kleene_channels() + return Column._unwrap_operand(operand), None + + def _kleene_channels(self): + return self._true, self._null + + def _wrap(self, true_expr, null_pred): + return NullableBoolExpr(true_expr, self._table, null_pred) + + def __and__(self, other): + ta, na = self._true, self._null + tb, nb = self._channels(other) + if nb is None: + # ``unknown & false`` is false, so a row stays unknown only where + # the two-valued side is true. + null = na & tb + else: + null = (na & (tb | nb)) | (nb & (ta | na)) + return self._wrap(ta & tb, null) + + def __rand__(self, other): + return self.__and__(other) + + def __or__(self, other): + ta, na = self._true, self._null + tb, nb = self._channels(other) + # Dual of ``&``: ``unknown | true`` is true, so a row stays unknown + # only where the other side is not definitely true. + null = na & ~tb if nb is None else (na & ~tb) | (nb & ~ta) + return self._wrap(ta | tb, null) + + def __ror__(self, other): + return self.__or__(other) + + def __xor__(self, other): + ta, na = self._true, self._null + tb, nb = self._channels(other) + # Neither operand can be ignored: an unknown always makes the result + # unknown, whatever the other side is. + null = na if nb is None else na | nb + return self._wrap((ta ^ tb) & ~null, null) + + def __rxor__(self, other): + return self.__xor__(other) + + def __invert__(self): + na = self._null + return self._wrap(~self._true & ~na, na) + + # ------------------------------------------------------------------ + # Reading the third value + # ------------------------------------------------------------------ + + def _logical(self, pred) -> np.ndarray: + """A physical boolean channel as one flag per live row.""" + arr = pred.compute() if isinstance(pred, blosc2.LazyExpr) else pred + return np.asarray(arr[:])[_live_positions(self._table)] + + def is_null(self) -> np.ndarray: + """Boolean array, True where this predicate's value is unknown. + + One flag per live row, matching :meth:`Column.is_null`. These are + exactly the rows ``where()`` drops without them being false. + """ + return self._logical(self._null) + + def notnull(self) -> np.ndarray: + """Boolean array, True where this predicate has a definite value.""" + return ~self.is_null() + + def null_count(self) -> int: + """How many live rows this predicate is unknown for.""" + return int(self.is_null().sum()) + + def fillna(self, value: bool): + """Resolve the unknown rows to *value*, giving a two-valued predicate. + + ``fillna(False)`` is what ``where()`` does implicitly (SQL ``WHERE`` + keeps only definite matches); ``fillna(True)`` is the opposite reading, + useful for "keep the rows I cannot rule out". The result is a plain + lazy predicate with no null channel left to propagate. + """ + if value: + return self._true | self._null + return self._true + + +def _sql_predicate(value): + """Collapse a three-valued predicate to the two-valued one ``WHERE`` wants. + + Every consumer of a boolean predicate goes through this: SQL ``WHERE`` + keeps the rows a predicate is *true* for and drops both the false and the + unknown ones, so the truth channel is the answer as-is. + """ + return value._true if isinstance(value, NullableBoolExpr) else value + + +def _live_positions(table) -> np.ndarray: + """Physical positions of *table*'s live rows, honouring a sorted view's order.""" + slp = getattr(table, "_cached_live_positions", None) + if slp is not None and table.base is not None: + return slp + return np.where(table._valid_rows[:])[0] + + class Column: """Column view for a :class:`CTable`, with vectorized operations and reductions.""" @@ -1304,7 +1520,11 @@ def _lazy_valid_rows(self): return self._table._valid_rows & self._mask def _resolve_live_positions(self) -> np.ndarray: - """Physical positions for all live rows, respecting sorted-view order.""" + """Physical positions for all live rows, respecting sorted-view order. + + Column-level, not table-level: a filtered column view carries its own + ``_mask``, which :func:`_live_positions` knows nothing about. + """ slp = getattr(self._table, "_cached_live_positions", None) if slp is not None and self._table.base is not None: return slp @@ -2072,14 +2292,43 @@ def _null_aware_arith(self, other, raw_result): return NullableExpr(blosc2.where(null_pred, np.nan, raw_result), self._table, null_pred) def _null_aware_compare(self, other, raw_result): - """SQL ``WHERE`` semantics for comparisons: a null - operand never satisfies any comparison, so null rows are forced to - False. Costs nothing when neither operand is nullable. + """Three-valued semantics for comparisons: a null operand makes the + comparison *unknown*, which SQL ``WHERE`` treats as no match. + + The returned :class:`NullableBoolExpr` reads as exactly that two-valued + predicate everywhere a boolean is wanted, and additionally remembers + which rows were unknown rather than false — so ``~`` can tell them + apart, which is the whole of Kleene logic here. Costs nothing when + neither operand is nullable: *raw_result* comes back unchanged. """ null_pred = self._combined_null_pred(other) if null_pred is None: return raw_result - return raw_result & ~null_pred + return NullableBoolExpr(raw_result & ~null_pred, self._table, null_pred) + + def _kleene_channels(self): + """``(definitely-true, unknown)`` channels for this column as a boolean + operand — what ``t.flag & (t.a > 10)`` combines. + + A nullable bool column has all three values in one array: the sentinel + one keeps them in band (``1``/``0``/``255``), the mask one puts the + third in its sidecar and leaves the ``False`` fill behind. Both are + read here so neither leaks a null into the truth channel. + """ + self._ensure_queryable() + raw = self._raw_col + if self._is_nullable_bool: + return raw == 1, self._nulls.null_pred() + if self.dtype != np.dtype(np.bool_): + # Not a boolean column: ``&`` on it is bitwise arithmetic, not + # logic, so there is no third value to look for. Tested before + # asking the null channel, which would otherwise build a predicate + # for every ``t.flags & 0x04``. + return raw, None + valid = self._nulls.valid_pred() + if valid is None: + return raw, None # non-nullable, or a mask column with no nulls yet + return raw & valid, ~valid def __neg__(self): self._ensure_queryable() @@ -2149,46 +2398,67 @@ def __rpow__(self, other): self._ensure_queryable() return self._null_aware_arith(other, self._unwrap_operand(other) ** self._raw_col) + def _as_kleene(self): + """This column as a three-valued predicate, or ``None`` if it is not one. + + Only a *nullable boolean* column carries a third value of its own; for + anything else the raw array is the whole story and the boolean + operators below stay exactly as cheap as they were. + """ + true, null = self._kleene_channels() + return None if null is None else NullableBoolExpr(true, self._table, null) + def __and__(self, other): self._ensure_queryable() + kleene = self._as_kleene() + if kleene is not None: + return kleene & other return self._raw_col & self._unwrap_operand(other) def __rand__(self, other): self._ensure_queryable() + kleene = self._as_kleene() + if kleene is not None: + return kleene & other return self._unwrap_operand(other) & self._raw_col def __or__(self, other): self._ensure_queryable() + kleene = self._as_kleene() + if kleene is not None: + return kleene | other return self._raw_col | self._unwrap_operand(other) def __ror__(self, other): self._ensure_queryable() + kleene = self._as_kleene() + if kleene is not None: + return kleene | other return self._unwrap_operand(other) | self._raw_col def __xor__(self, other): self._ensure_queryable() + kleene = self._as_kleene() + if kleene is not None: + return kleene ^ other return self._raw_col ^ self._unwrap_operand(other) def __rxor__(self, other): self._ensure_queryable() + kleene = self._as_kleene() + if kleene is not None: + return kleene ^ other return self._unwrap_operand(other) ^ self._raw_col def __invert__(self): self._ensure_queryable() - if self._is_nullable_bool: - return self._raw_col == 0 - inverted = ~self._raw_col - if self.dtype == np.dtype(np.bool_): - valid = self._nulls.valid_pred() - if valid is not None: - # A mask-backed nullable bool holds the ``False`` fill in its - # null rows, and ``~False`` is True -- so without this, negating - # a nullable flag *selects* its nulls. SQL WHERE semantics: a - # null satisfies neither the predicate nor its negation. The - # sentinel branch above gets this from ``== 0``, which excludes - # the reserved 255 as well as the true rows. - return inverted & valid - return inverted + kleene = self._as_kleene() + if kleene is not None: + # A null flag negates to a null, not to a match -- both storages + # would otherwise say True for it: the mask one holds the ``False`` + # fill, and the sentinel one holds 255, which is neither 0 nor 1. + return ~kleene + return ~self._raw_col def __lt__(self, other): if self.is_utf8: @@ -2308,7 +2578,7 @@ def fn(chunk, start, stop): return res raw = self._utf8_chunked_bool(fn) - return blosc2.asarray(raw) & self._lazy_valid_rows() + return self._utf8_kleene(blosc2.asarray(raw) & self._lazy_valid_rows(), other) def _utf8_scalar_mask(self, numpy_op, value: str) -> np.ndarray: """Raw physical-length boolean mask for ``column value``. @@ -2420,7 +2690,48 @@ def _utf8_index_mask(self, numpy_op, value: str) -> np.ndarray | None: def _utf8_compare_scalar(self, numpy_op, value: str): """Scalar comparison as a live-row-intersected boolean NDArray.""" - return blosc2.asarray(self._utf8_scalar_mask(numpy_op, value)) & self._lazy_valid_rows() + true = blosc2.asarray(self._utf8_scalar_mask(numpy_op, value)) & self._lazy_valid_rows() + return self._utf8_kleene(true) + + def _utf8_null_phys(self) -> np.ndarray | None: + """Physical-length boolean array, True where this utf8 column is null. + + ``None`` when the column has no nulls to find. Unlike the other kinds, + utf8 cannot answer this as a lazy predicate — neither a + ``StringDType`` comparison nor a raw-bytes span scan is something the + lazy layer evaluates — so it is a materialized array, and one the + caller pays a scan for. That is why :class:`NullableBoolExpr` accepts + a deferred null channel: only ``~`` and :meth:`is_null` need this. + """ + n_phys = len(self._table._valid_rows) + valid = self._nulls.valid_array() + if valid is not None: + out = np.ones(n_phys, dtype=bool) + flags = np.asarray(valid[:], dtype=bool)[:n_phys] + out[: len(flags)] = flags + return ~out + nv = self.null_value + if nv is None: + return None + return self._utf8_chunked_bytes(lambda arr, start, stop: arr.equal_mask_span(nv, start, stop)) + + def _utf8_kleene(self, true_pred, other: Column | None = None): + """Attach the deferred null channel to a utf8 comparison result.""" + columns = [c for c in (self, other) if c is not None and c._nulls.is_nullable] + if not columns: + return true_pred + + def null_channel(): + combined = None + for col in columns: + part = col._utf8_null_phys() + if part is not None: + combined = part if combined is None else (combined | part) + if combined is None: + combined = np.zeros(len(self._table._valid_rows), dtype=bool) + return blosc2.asarray(combined) + + return NullableBoolExpr(true_pred, self._table, null_channel) def _dictionary_index_mask(self, value: str) -> np.ndarray | None: """Answer ``column == value`` from the dict-rank index, or ``None``. @@ -2474,24 +2785,49 @@ def _dictionary_eq(self, other, *, negate: bool = False): *negate* inverts the value test *before* the live-row intersection, so ``!=`` stays a same-shaped predicate over live rows. Negating the returned value instead would turn every dead slot True. + + ``other is None`` is the *is-null* test rather than a comparison + against a value, so it alone answers two-valued; every other form + returns a three-valued :class:`NullableBoolExpr`, because a row with + the reserved null code equals no string and differs from none either. """ n_phys = len(self._table._valid_rows) dc = self._raw_col # DictionaryColumn spec = self._table._schema.columns_by_name[self._col_name].spec valid = self._lazy_valid_rows() + + def kleene(pred, *, exclude_nulls: bool): + """Wrap a value test, whose null rows are neither true nor false. + + *exclude_nulls* says whether the truth channel still has to have + them taken out. It is False for a positive test — no null row + carries the code being looked for, so ``==`` is already exact — + and True for every negated one, where the reserved code differs + from the target like any other and would otherwise read as a + match. The null channel itself is deferred, so a query that only + ever asks ``==`` never reads the codes for it. + """ + + def null(): + return self._dictionary_null_pred(dc, spec, n_phys) + + if exclude_nulls: + pred = pred & ~null() + return NullableBoolExpr(pred, self._table, null) + if other is None: target_code = spec.null_code elif isinstance(other, str): indexed = self._dictionary_index_mask(other) if indexed is not None: - return blosc2.asarray(~indexed if negate else indexed) & valid + return kleene(blosc2.asarray(~indexed if negate else indexed) & valid, exclude_nulls=negate) try: target_code = dc.value_to_code(other) except KeyError: # No row carries this value: nothing matches, everything differs. if negate: - return blosc2.ones(n_phys, dtype=np.bool_) & valid - return blosc2.zeros(n_phys, dtype=np.bool_) + return kleene(blosc2.ones(n_phys, dtype=np.bool_) & valid, exclude_nulls=True) + return kleene(blosc2.zeros(n_phys, dtype=np.bool_), exclude_nulls=False) else: raise TypeError( f"Dictionary column {self._col_name!r} can only be compared with str or None, " @@ -2503,7 +2839,17 @@ def _dictionary_eq(self, other, *, negate: bool = False): physical = blosc2.zeros(n_phys, dtype=np.bool_) physical[: len(dc.codes)] = pred pred = physical - return pred & valid + pred = pred & valid + return pred if other is None else kleene(pred, exclude_nulls=negate) + + def _dictionary_null_pred(self, dc, spec, n_phys: int): + """Physical-slot predicate, True where this dictionary column is null.""" + null = dc.codes == np.int32(spec.null_code) + if len(dc.codes) != n_phys: + physical = blosc2.zeros(n_phys, dtype=np.bool_) + physical[: len(dc.codes)] = null + null = physical + return null def isin(self, values) -> np.ndarray: """Return a boolean array True where the live value is in *values*. @@ -2521,6 +2867,12 @@ def isin(self, values) -> np.ndarray: ``NaN`` is **not** one of them -- under mask storage NaN is an ordinary value (decision 6 of the mask-nulls design), so it is matched as data. + Deliberately **two-valued**, unlike a comparison (see + :class:`NullableBoolExpr`): this returns a materialized array, and the + ``None`` spelling above already answers the question a third value + would. So ``~col.isin([...])`` includes the null rows, where + ``~(col == x)`` excludes them. + For dictionary columns this performs efficient integer-code membership testing (no decoding of all values). Values absent from the dictionary are treated as not-present. @@ -2980,6 +3332,7 @@ def _require_kind(self, kinds: str, op: str) -> None: def _normalize_sum_where(self, where): """Normalize an optional ``sum(where=...)`` predicate to a boolean array/expression.""" + where = _sql_predicate(where) if where is None: return None if isinstance(where, str): @@ -12613,6 +12966,11 @@ def _coerce_assign_operand(value): machinery accepts: a LazyExpr, DSLKernel, callable, or string.""" if isinstance(value, NullableExpr): return value._expr + if isinstance(value, NullableBoolExpr): + # A computed column is a stored array of values: it has nowhere to + # carry a third one, so the unknown rows land as False -- the same + # collapse ``where()`` makes. + return value._true if isinstance(value, Column): value._ensure_queryable() raw = value._raw_col @@ -12890,6 +13248,7 @@ def __getitem__(self, key): """ if isinstance(key, ColExpr): key = key._bind(self) + key = _sql_predicate(key) if isinstance(key, str): physical = self._logical_to_physical_name(key) if physical in self._cols or physical in self._computed_cols: @@ -15174,6 +15533,15 @@ def where( # noqa: C901 Notes ----- + A predicate over a **nullable** column is three-valued: a comparison + against a null is *unknown*, not false, and ``&``/``|``/``~`` combine + that third value by Kleene's rules (see :class:`NullableBoolExpr`). + ``where()`` keeps the rows a predicate is **true** for, so an unknown + row is dropped just like a false one -- which is what makes + ``t.where(~(t.a > 10))`` exclude the nulls rather than select them. + Both spellings below agree, and either predicate can be asked which + rows it could not answer for (``p.is_null()``). + Use bitwise operators (``&``, ``|``, ``~``) or string expressions for element-wise boolean logic. Python's logical operators ``and``, ``or`` and ``not`` cannot be overloaded and therefore do not build lazy column @@ -15192,6 +15560,10 @@ def where( # noqa: C901 """ if isinstance(expr_result, ColExpr): expr_result = expr_result._bind(self) + # SQL WHERE keeps the rows a predicate is *true* for: a three-valued + # predicate contributes its truth channel and drops the unknown rows, + # which is what makes ``~`` correct without any change here. + expr_result = _sql_predicate(expr_result) if isinstance(expr_result, str): self._guard_varlen_scalar_expression(expr_result, allow_utf8=True) utf8_names = self._utf8_names_in(expr_result) diff --git a/src/blosc2/ctable_indexing.py b/src/blosc2/ctable_indexing.py index 6ba285dea..755c725dc 100644 --- a/src/blosc2/ctable_indexing.py +++ b/src/blosc2/ctable_indexing.py @@ -182,8 +182,17 @@ def _persist_utf8_vocab(full: dict, meta: dict, sorted_vocab: np.ndarray) -> Non meta["vocab_path"] = vocab_path -def _expression_has_or(expression: str) -> bool: - """Whether *expression* really contains a boolean OR. +def _expression_defeats_global_null_filter(expression: str) -> bool: + """Whether *expression* really contains a boolean OR, or a negation. + + Both make a *global* null post-filter wrong, for the same underlying + reason: the predicate's answer for a null row is not simply "no match". + Under OR a row that is null in one column can still match the other + branch; under Kleene negation ``~(unknown & false)`` is *true*, so a row + the filter would drop qualifies (see + ``blosc2.ctable_nulls._NullPredicateRewriter._channels``). Where the + predicate is actually evaluated -- the segment path -- both are already + exact per leaf and the filter is not merely wrong but unnecessary. A substring test for ``"|"`` also fires on a string literal that happens to contain one (``name == 'a|b'``), and the answer decides whether a nullable @@ -199,6 +208,8 @@ def _expression_has_or(expression: str) -> bool: return True if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): return True + if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.Not, ast.Invert)): + return True return False @@ -1580,28 +1591,32 @@ def _try_index_where(self, expr_result: blosc2.LazyExpr) -> np.ndarray | None: and not is_nan_sentinel(root._schema.columns_by_name[name].spec.null_value) ] - # Global null post-filtering is not correct for OR expressions: it would - # drop a row that is null in one column but matches the other branch. - # Which paths that rules out depends on *how* each one answers. + # Global null post-filtering is not correct for an OR expression -- it + # would drop a row that is null in one column but matches the other + # branch -- nor for a negation, where ``~(unknown & false)`` is true and + # the row it would drop qualifies. Which paths that rules out depends + # on *how* each one answers. # # The segment/candidate-unit path evaluates the predicate through # miniexpr over the surviving blocks, and the predicate is null-aware # per leaf by the time it gets here (CTable._rewrite_null_predicates for # the string form, Column._null_aware_compare for the operator one), so # its result is already exact and the post-filter is not merely - # incorrect for OR but unnecessary. That path therefore serves OR, with - # the filter skipped. + # incorrect for those shapes but unnecessary. That path therefore + # serves them, with the filter skipped. # # The exact-position paths (FULL/PARTIAL/BUCKET) answer by taking an # ordered *range* of the sorted column and never evaluate anything, so - # the post-filter is load-bearing there and an OR over a nullable - # indexed column still falls back to the scan -- which is itself - # null-aware, and so returns the right answer. Those bails are - # belt-and-braces today: the planner declines an exact plan for any - # expression containing an OR, so such a plan cannot reach them. They - # are what stops a planner that learns to build one from silently - # reintroducing the bug. - skip_null_filter = bool(nullable_indexed) and _expression_has_or(expr_result.expression) + # the post-filter is load-bearing there and such an expression over a + # nullable indexed column still falls back to the scan -- which is + # itself null-aware, and so returns the right answer. Those bails are + # belt-and-braces today: the planner declines an exact plan for an + # expression containing an OR or a negation, so such a plan cannot reach + # them. They are what stops a planner that learns to build one from + # silently reintroducing the bug. + skip_null_filter = bool(nullable_indexed) and _expression_defeats_global_null_filter( + expr_result.expression + ) # Inject every usable table-owned descriptor so plan_query can combine them. # In .b2z read mode all columns share the same urlpath, so _array_key() diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index efdf01f4c..dc0009734 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -23,6 +23,7 @@ from __future__ import annotations import ast +import copy import math import operator from builtins import bool as builtin_bool @@ -324,48 +325,90 @@ def _sentinel_can_match(node: ast.Compare, name: str, sentinel) -> bool: return True +def _and(left: ast.AST, right: ast.AST) -> ast.AST: + return ast.BinOp(left=copy.deepcopy(left), op=ast.BitAnd(), right=copy.deepcopy(right)) + + +def _or(left: ast.AST, right: ast.AST) -> ast.AST: + return ast.BinOp(left=copy.deepcopy(left), op=ast.BitOr(), right=copy.deepcopy(right)) + + +def _invert(node: ast.AST) -> ast.AST: + # Fold ``~~x`` away: a guard reaches here already negated once (the unknown + # channel of a leaf is ``~guard``), and negating it back is common enough + # that leaving the pair in would show up in every rewritten negation. + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Invert): + return copy.deepcopy(node.operand) + return ast.UnaryOp(op=ast.Invert(), operand=copy.deepcopy(node)) + + +def _is_negation(node: ast.AST) -> builtin_bool: + return isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.Not, ast.Invert)) + + +def _boolean_parts(node: ast.AST): + """``(op, values)`` for a boolean combination, or ``None``. + + Reads ``and``/``or`` and ``&``/``|`` as the same two operators, since the + rewrite normalizes the first pair into the second anyway. + """ + if isinstance(node, ast.BoolOp): + return ("&" if isinstance(node.op, ast.And) else "|"), list(node.values) + if isinstance(node, ast.BinOp) and isinstance(node.op, (ast.BitAnd, ast.BitOr)): + return ("&" if isinstance(node.op, ast.BitAnd) else "|"), [node.left, node.right] + return None + + class _NullPredicateRewriter(ast.NodeTransformer): - """Conjoin a validity guard onto every predicate over a nullable column.""" + """Conjoin a validity guard onto every predicate over a nullable column. + + Outside a negation that is the whole job, and the result is the plain + ``(a > 10) & (a != 999)`` the index planner already knows how to serve. + Under a negation the rewriter switches to :meth:`_channels`, which carries + a second, *unknown* channel alongside the truth one so that ``~`` can tell + a null row from a false one -- Kleene's logic, spelled out in AST. + """ def __init__(self, guards: dict[str, tuple[str, object]]) -> None: self._guards = guards - self._in_negation = 0 self.changed = False + def _guard_text(self, name: str) -> ast.AST: + """Freshly parsed guard AST for *name* (nodes must never be shared).""" + return ast.parse(self._guards[name][0], mode="eval").body + def _guard(self, node: ast.AST, names: set[str]) -> ast.AST: - for guard in sorted({self._guards[n][0] for n in names if n in self._guards}): - # Re-parse per insertion: the same guard can appear at several - # leaves, and an AST node must not be shared between them. - right = ast.parse(guard, mode="eval").body - node = ast.BinOp(left=node, op=ast.BitAnd(), right=right) + for name in sorted(names): + node = ast.BinOp(left=node, op=ast.BitAnd(), right=self._guard_text(name)) self.changed = True return node - def visit_Compare(self, node: ast.Compare) -> ast.AST: - if self._in_negation: - # The enclosing negation already conjoins validity for every column - # in its subtree, and that outer conjunct dominates whatever this - # leaf evaluates to for a null row. Guarding here as well would be - # redundant, and it would put the guard on the wrong side of the - # ``~`` (see the note in :func:`rewrite_null_predicates`). It would - # also be wrong to skip it via _sentinel_can_match: a sentinel that - # fails ``a > 10`` *passes* ``not (a > 10)``. - return node - names = { + def _guarded_names(self, node: ast.Compare) -> set[str]: + """Nullable columns whose null could satisfy this leaf, so it needs a guard. + + The others are left alone deliberately: an unguarded single-predicate + expression is the shape the index planner recognizes, so a needless + guard costs a query its index (see :func:`_sentinel_can_match`). + """ + return { n for n in _collect_names(node) if n in self._guards and _sentinel_can_match(node, n, self._guards[n][1]) } - return self._guard(node, names) + + def visit_Compare(self, node: ast.Compare) -> ast.AST: + return self._guard(node, self._guarded_names(node)) def visit_UnaryOp(self, node: ast.UnaryOp) -> ast.AST: - if not isinstance(node.op, (ast.Not, ast.Invert)): + if not _is_negation(node): return self.generic_visit(node) - names = _collect_names(node.operand) - self._in_negation += 1 - node = self.generic_visit(node) - self._in_negation -= 1 - return self._guard(node, names) + true, null = self._channels(node) + if null is not None: + # ``~unknown`` is unknown, and an unknown row is not a match: the + # negation's own truth channel already excludes it, so nothing more + # is conjoined here. Returning the truth channel *is* the answer. + self.changed = True + return true def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: """Normalize ``and``/``or`` to ``&``/``|``. @@ -382,6 +425,96 @@ def visit_BoolOp(self, node: ast.BoolOp) -> ast.AST: combined = ast.BinOp(left=combined, op=op, right=value) return combined + # ------------------------------------------------------------------ + # Kleene channels -- built only under a negation + # ------------------------------------------------------------------ + + def _channels(self, node: ast.AST) -> tuple[ast.AST, ast.AST | None]: + """``(definitely-true, unknown)`` ASTs for a boolean subtree. + + The unknown channel is ``None`` when no nullable column takes part, + and every rule below reads that as "no row is unknown" -- which + collapses the whole construction back to two-valued logic, so an + expression that meets no null pays nothing for this. + """ + if _is_negation(node): + if isinstance(node.operand, ast.Compare): + # ``~(leaf & guard) & guard`` is just ``~leaf & guard``: where + # the guard holds it is redundant, and where it does not the + # outer conjunct decides. Worth the special case -- negating a + # single leaf is the common shape, and this keeps it the short + # form the index planner has always seen. + null = self._leaf_null(node.operand) + negated = ast.UnaryOp(op=node.op, operand=node.operand) + return (negated, None) if null is None else (_and(negated, _invert(null)), null) + true, null = self._channels(node.operand) + negated = ast.UnaryOp(op=node.op, operand=true) + if null is None: + return negated, None + # A row that was unknown stays unknown, and so is not a match: + # ``~true & ~unknown``. This is where two-valued negation went + # wrong -- it inverted a null that had already been collapsed to + # False and turned it into a match. + return _and(negated, _invert(null)), null + + parts = _boolean_parts(node) + if parts is not None: + op, values = parts + true, null = self._channels(values[0]) + for value in values[1:]: + true, null = self._combine(op, true, null, *self._channels(value)) + return true, null + + if isinstance(node, ast.Compare): + return self._guard(node, self._guarded_names(node)), self._leaf_null(node) + + # Anything else (a bare boolean column, a function call): two-valued as + # far as this rewrite can tell, and visited so nested leaves still get + # their guards. + return self.visit(node), None + + def _leaf_null(self, node: ast.Compare) -> ast.AST | None: + """The *unknown* channel of one comparison: null in any operand it reads. + + Every nullable operand contributes, including the ones whose guard the + truth channel is allowed to skip (:meth:`_guarded_names`): a leaf a + sentinel could never satisfy is still *unknown* for that row rather + than false, and under a negation that is exactly the difference. + """ + null = None + for name in sorted(_collect_names(node) & set(self._guards)): + term = _invert(self._guard_text(name)) + null = term if null is None else _or(null, term) + return null + + @staticmethod + def _combine(op, ta, na, tb, nb): + """One Kleene ``&``/``|`` step over two ``(true, unknown)`` pairs.""" + if op == "&": + true = _and(ta, tb) + if na is None and nb is None: + null = None + elif nb is None: + # ``unknown & false`` is false: only a true other side leaves + # the conjunction unknown. + null = _and(na, tb) + elif na is None: + null = _and(nb, ta) + else: + null = _or(_and(na, _or(tb, nb)), _and(nb, _or(ta, na))) + return true, null + true = _or(ta, tb) + if na is None and nb is None: + null = None + elif nb is None: + # Dual: ``unknown | true`` is true. + null = _and(na, _invert(tb)) + elif na is None: + null = _and(nb, _invert(ta)) + else: + null = _or(_and(na, _invert(tb)), _and(nb, _invert(ta))) + return true, null + def sentinel_guard_expr(name: str, null_value) -> str | None: """Source text for a predicate that is True where *name* is not null. @@ -422,14 +555,15 @@ def rewrite_null_predicates(expr: str, guards: dict[str, tuple[str, object]]) -> is null in ``a`` but matches the other branch of ``(a > 10) | (b == 0)``, where SQL says the row qualifies. - Under a negation the guard is attached at the negation itself, not inside - it: ``not (a > 10)`` must be False for a null ``a``, but ``~((a > 10) & - guard)`` would yield True. Guarding outside -- ``(~(a > 10)) & guard`` -- - gives False. This is exact for every form the tests cover, and - conservative in one three-valued corner: ``not (a > 10 and b == 999)`` - where the second term is False makes SQL's ``NULL AND FALSE`` collapse to - ``FALSE``, so the row should survive the negation, while the guard drops - it. Rows are only ever dropped, never wrongly returned. + Under a negation the rewrite switches to Kleene's three-valued logic, + carrying an *unknown* channel beside the truth one (see + :meth:`_NullPredicateRewriter._channels`). ``~`` cannot be handled by + conjoining a guard anywhere: put it inside and ``~((a > 10) & guard)`` + turns the null into a match; put it outside and ``(~(a > 10)) & guard`` + drops rows SQL keeps, because ``NULL AND FALSE`` is ``FALSE`` and its + negation is therefore true. The two channels give the exact answer for + both, at the cost of a longer expression -- paid only where a negation + actually meets a nullable column. Returns the rewritten expression, or ``None`` when nothing was rewritten -- including when *expr* does not parse, which leaves the caller's original diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index 6b01c123d..57620b6c7 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -551,6 +551,20 @@ def _normalize_expr_operand(value: Any) -> Any: return raw_col if raw_col is not None else value +def _defers_boolean_op(value: Any) -> bool: + """True when *value* implements a richer boolean algebra than two values. + + A CTable predicate over a nullable column is three-valued (see + :class:`blosc2.ctable.NullableBoolExpr`): besides true and false it has + *unknown*, and ``&``/``|``/``^`` combine that third value by Kleene's + rules rather than bitwise ones. Returning ``NotImplemented`` from the + two-valued operator hands the operation to the operand that knows those + rules, so ``lazy_expr & nullable_predicate`` stays three-valued instead of + silently collapsing to whichever side Python happened to evaluate first. + """ + return hasattr(value, "_kleene_channels") + + def _check_allowed_dtypes( value: bool | int | float | str | blosc2.Array, ): @@ -3485,6 +3499,8 @@ def __abs__(self) -> blosc2.LazyExpr: @is_documented_by(bitwise_and) def __and__(self, value: int | float | blosc2.Array, /) -> blosc2.LazyExpr: + if _defers_boolean_op(value): + return NotImplemented value = _normalize_expr_operand(value) _check_allowed_dtypes(value) return blosc2.LazyExpr(new_op=(self, "&", value)) @@ -3497,6 +3513,8 @@ def __rand__(self, value: int | float | blosc2.Array, /) -> blosc2.LazyExpr: @is_documented_by(bitwise_xor) def __xor__(self, other) -> blosc2.LazyExpr: + if _defers_boolean_op(other): + return NotImplemented other = _normalize_expr_operand(other) _check_allowed_dtypes(other) return blosc2.LazyExpr(new_op=(self, "^", other)) @@ -3509,6 +3527,8 @@ def __rxor__(self, other) -> blosc2.LazyExpr: @is_documented_by(bitwise_or) def __or__(self, other) -> blosc2.LazyExpr: + if _defers_boolean_op(other): + return NotImplemented other = _normalize_expr_operand(other) _check_allowed_dtypes(other) return blosc2.LazyExpr(new_op=(self, "|", other)) diff --git a/tests/ctable/test_kleene_logic.py b/tests/ctable/test_kleene_logic.py new file mode 100644 index 000000000..cd43af5dd --- /dev/null +++ b/tests/ctable/test_kleene_logic.py @@ -0,0 +1,520 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Three-valued (Kleene) logic for predicates over nullable columns. + +A comparison against a null is neither true nor false. Every earlier phase +collapsed that third value to False at the leaf, which is right for ``WHERE`` +and wrong for ``~``: negating a collapsed null turns it into a match. The +comparison now returns a :class:`blosc2.ctable.NullableBoolExpr` carrying the +unknown rows beside the true ones, and ``&``/``|``/``^``/``~`` combine them by +Kleene's rules. + +The load-bearing case is ``unknown & false``, which is **false**, not unknown +-- no value of the missing operand could make the conjunction true. That is +why "drop every row a null touched" is not an implementation of this, and it +is what ``test_and_or_truth_tables`` pins. + +Both query forms are covered, because they get there by different routes: the +operator form through the expression objects, the string form through an AST +rewrite that carries two channels under a negation +(``ctable_nulls._NullPredicateRewriter._channels``). They are checked against +each other and against a NumPy oracle, over both null storages and with and +without indexes. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pytest + +import blosc2 +from blosc2.ctable import NullableBoolExpr + +# --------------------------------------------------------------------------- +# A Kleene oracle, written the way the truth tables read +# --------------------------------------------------------------------------- + +TRUE, FALSE, UNKNOWN = True, False, None +CELLS = (TRUE, FALSE, UNKNOWN) + + +def k_and(x, y): + if x is FALSE or y is FALSE: + return FALSE + return UNKNOWN if UNKNOWN in (x, y) else TRUE + + +def k_or(x, y): + if x is TRUE or y is TRUE: + return TRUE + return UNKNOWN if UNKNOWN in (x, y) else FALSE + + +def k_xor(x, y): + return UNKNOWN if UNKNOWN in (x, y) else (x != y) + + +def k_not(x): + return UNKNOWN if x is UNKNOWN else (not x) + + +def observed(t, pred): + """The predicate's three-valued result, one cell per live row. + + Read the way a user would: a row is true when ``where()`` keeps it and + unknown when the predicate says so. Rows the query dropped without being + unknown are the false ones. + """ + n = t.nrows + unknown = pred.is_null() if isinstance(pred, NullableBoolExpr) else np.zeros(n, dtype=bool) + true = np.zeros(n, dtype=bool) + true[np.asarray(t.where(pred)["id"][:], dtype=int)] = True + return [UNKNOWN if u else bool(v) for v, u in zip(true, unknown, strict=True)] + + +# --------------------------------------------------------------------------- +# Fixtures: one row per cell of the truth table +# --------------------------------------------------------------------------- + +PAIRS = [(x, y) for x in CELLS for y in CELLS] +STORAGES = ("mask", "sentinel") + + +def nullable(factory, storage, sentinel): + """A nullable spec plus the value that spells "null" for it. + + A sentinel column has no way to write a null other than its sentinel, so + the two storages need different cells for the same logical row -- which is + the whole reason both are parametrized here rather than assumed alike. + """ + if storage == "mask": + return factory(nullable=True, null_storage="mask"), None + return factory(nullable=True, null_value=sentinel), sentinel + + +def bool_pair_table(storage): + """Two nullable bool columns holding every ``(x, y)`` combination.""" + spec, null_cell = nullable(blosc2.bool, storage, 255) + Row = dataclasses.make_dataclass( + "R", + [ + ("id", int, blosc2.field(blosc2.int64())), + ("p", bool, blosc2.field(spec)), + ("q", bool, blosc2.field(spec)), + ], + ) + cell = {TRUE: True, FALSE: False, UNKNOWN: null_cell} + return blosc2.CTable(Row, new_data=[(i, cell[x], cell[y]) for i, (x, y) in enumerate(PAIRS)]) + + +def int_pair_table(storage): + """The same combinations as ``a == 1`` / ``b == 1`` over nullable ints.""" + spec, null_cell = nullable(blosc2.int64, storage, -9) + Row = dataclasses.make_dataclass( + "R", + [ + ("id", int, blosc2.field(blosc2.int64())), + ("a", int, blosc2.field(spec)), + ("b", int, blosc2.field(spec)), + ], + ) + cell = {TRUE: 1, FALSE: 0, UNKNOWN: null_cell} + return blosc2.CTable(Row, new_data=[(i, cell[x], cell[y]) for i, (x, y) in enumerate(PAIRS)]) + + +# --------------------------------------------------------------------------- +# The truth tables themselves +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("storage", STORAGES) +@pytest.mark.parametrize( + ("op", "expected"), + [("&", k_and), ("|", k_or), ("^", k_xor)], +) +def test_and_or_truth_tables(storage, op, expected): + """All nine cells, for both ways of spelling a three-valued operand. + + ``unknown & false`` is the cell that matters: it must come out **false**. + A row is dropped by ``where()`` either way, so only the unknown channel + tells the two apart -- which is exactly what makes an enclosing ``~`` + return the right answer. + """ + t = int_pair_table(storage) + a, b = (t.a == 1), (t.b == 1) + pred = {"&": a & b, "|": a | b, "^": a ^ b}[op] + assert observed(t, pred) == [expected(x, y) for x, y in PAIRS] + + +@pytest.mark.parametrize("storage", STORAGES) +def test_not_truth_table(storage): + t = int_pair_table(storage) + assert observed(t, ~(t.a == 1)) == [k_not(x) for x, _ in PAIRS] + + +@pytest.mark.parametrize("storage", STORAGES) +@pytest.mark.parametrize("op", ["&", "|", "^"]) +def test_nullable_bool_columns_are_three_valued_operands(storage, op): + """A nullable bool column *is* a three-valued predicate -- no comparison + needed. Its third value is in band for sentinel storage (the reserved + ``255``) and in the sidecar for mask storage; both are read here.""" + t = bool_pair_table(storage) + pred = {"&": t.p & t.q, "|": t.p | t.q, "^": t.p ^ t.q}[op] + expected = {"&": k_and, "|": k_or, "^": k_xor}[op] + assert observed(t, pred) == [expected(x, y) for x, y in PAIRS] + + +@pytest.mark.parametrize("storage", STORAGES) +def test_negating_a_nullable_bool_column_keeps_its_nulls_out(storage): + t = bool_pair_table(storage) + assert observed(t, ~t.p) == [k_not(x) for x, _ in PAIRS] + + +@pytest.mark.parametrize("storage", STORAGES) +def test_double_negation_is_the_identity(storage): + """Kleene negation is an involution -- ``~~p`` is ``p``, unknown included. + Two-valued negation was not: it lost the null on the first ``~``.""" + t = int_pair_table(storage) + p = t.a == 1 + assert observed(t, ~(~p)) == observed(t, p) + + +@pytest.mark.parametrize("storage", STORAGES) +def test_de_morgan_holds(storage): + t = int_pair_table(storage) + a, b = (t.a == 1), (t.b == 1) + assert observed(t, ~(a & b)) == observed(t, (~a) | (~b)) + assert observed(t, ~(a | b)) == observed(t, (~a) & (~b)) + + +@pytest.mark.parametrize("storage", STORAGES) +def test_excluded_middle_does_not_hold(storage): + """``p | ~p`` is *not* a tautology when p is unknown -- the property that + separates three-valued logic from two-valued. Pinned so that a future + "simplification" of the operators cannot quietly restore it.""" + t = int_pair_table(storage) + p = t.a == 1 + assert observed(t, p | ~p) == [k_or(x, k_not(x)) for x, _ in PAIRS] + assert UNKNOWN in observed(t, p | ~p) + + +# --------------------------------------------------------------------------- +# Mixing three-valued and two-valued operands +# --------------------------------------------------------------------------- + + +def test_a_two_valued_operand_combines_without_a_null_channel(): + """A non-nullable column contributes no unknowns, so the rules degrade to + the two-valued ones -- and the predicate over it is a plain lazy + expression, not a three-valued one, so nothing is paid for it.""" + t = int_pair_table("mask") + plain = t.id < 3 + assert not isinstance(plain, NullableBoolExpr) + assert isinstance(plain, blosc2.LazyExpr) + + a = t.a == 1 + combined = a & plain + assert observed(t, combined) == [k_and(x, i < 3) for i, (x, _y) in enumerate(PAIRS)] + + +def test_operand_order_does_not_change_the_answer(): + """``plain & three_valued`` must not silently collapse to two values. + + Python offers the reflected operator to a subclass first, which is why + :class:`NullableBoolExpr` subclasses :class:`blosc2.LazyExpr`; a + :class:`blosc2.NDArray` on the left defers explicitly instead + (``ndarray._defers_boolean_op``). + """ + t = int_pair_table("mask") + a = t.a == 1 + plain = t.id < 3 + assert isinstance(plain & a, NullableBoolExpr) + assert observed(t, plain & a) == observed(t, a & plain) + assert observed(t, plain | a) == observed(t, a | plain) + + materialized = blosc2.asarray(np.asarray(plain[:])) + assert isinstance(materialized & a, NullableBoolExpr) + assert observed(t, materialized & a) == observed(t, a & plain) + + +def test_a_column_on_the_right_of_a_lazy_operand_stays_three_valued(): + t = bool_pair_table("mask") + plain = t.id < 3 + assert isinstance(plain & t.p, NullableBoolExpr) + assert observed(t, plain & t.p) == [k_and(i < 3, x) for i, (x, _y) in enumerate(PAIRS)] + + +# --------------------------------------------------------------------------- +# Reading the third value +# --------------------------------------------------------------------------- + + +def test_is_null_notnull_and_null_count(): + t = int_pair_table("mask") + p = t.a == 1 + unknown = np.array([x is UNKNOWN for x, _ in PAIRS]) + np.testing.assert_array_equal(p.is_null(), unknown) + np.testing.assert_array_equal(p.notnull(), ~unknown) + assert p.null_count() == int(unknown.sum()) + + +def test_fillna_resolves_the_unknown_rows(): + """The two readings of an unknown row, both spelled explicitly. + + ``where()`` takes ``fillna(False)`` implicitly, which is SQL; the other + direction is "keep what I cannot rule out".""" + t = int_pair_table("mask") + p = ~(t.a == 1) + kept_false = sorted(t.where(p.fillna(False))["id"][:].tolist()) + kept_true = sorted(t.where(p.fillna(True))["id"][:].tolist()) + assert kept_false == [i for i, (x, _) in enumerate(PAIRS) if k_not(x) is TRUE] + assert kept_true == [i for i, (x, _) in enumerate(PAIRS) if k_not(x) is not FALSE] + assert not isinstance(p.fillna(True), NullableBoolExpr) + + +def test_the_predicate_is_a_lazy_expression(): + """It does not merely quack like one: consumers type-check, index with and + plan around this object, and it has to keep working for all of them.""" + t = int_pair_table("mask") + p = t.a == 1 + assert isinstance(p, blosc2.LazyExpr) + assert p.dtype == np.bool_ + computed = np.asarray(p.compute()[:]) + live = np.asarray(p[:])[: t.nrows] + np.testing.assert_array_equal(live, computed[: t.nrows]) + # The values are the collapsed two-valued predicate: unknown reads False. + np.testing.assert_array_equal(live, [x is TRUE for x, _ in PAIRS]) + + +# --------------------------------------------------------------------------- +# Every V1 kind negates the same way +# --------------------------------------------------------------------------- + +KIND_CASES = [ + ("int64", blosc2.int64, int, [1, 20, None, 30], 10, -9), + ("float64", blosc2.float64, float, [1.0, 20.0, None, 30.0], 10.0, float("nan")), + ( + "string", + lambda **kw: blosc2.string(max_length=8, **kw), + str, + ["aa", "zz", None, "yy"], + "mm", + "ZZZZZZZZ", + ), + ( + "bytes", + lambda **kw: blosc2.bytes(max_length=8, **kw), + bytes, + [b"aa", b"zz", None, b"yy"], + b"mm", + b"ZZZZZZZZ", + ), + ("utf8", blosc2.utf8, str, ["aa", "zz", None, "yy"], "mm", "__BLOSC2_NULL__"), +] + + +@pytest.mark.parametrize("storage", STORAGES) +@pytest.mark.parametrize(("label", "factory", "annotation", "values", "threshold", "sentinel"), KIND_CASES) +def test_negation_drops_nulls_for_every_kind( + label, factory, annotation, values, threshold, sentinel, storage +): + """Whatever the kind, ``~(a > x)`` keeps only rows that are definitely not + greater -- never the ones with nothing to compare.""" + spec, null_cell = nullable(factory, storage, sentinel) + Row = dataclasses.make_dataclass( + "R", + [ + ("id", int, blosc2.field(blosc2.int64())), + ("a", annotation, blosc2.field(spec)), + ], + ) + cells = [null_cell if v is None else v for v in values] + t = blosc2.CTable(Row, new_data=list(enumerate(cells))) + assert sorted(t.where(~(t.a > threshold))["id"][:].tolist()) == [0] + assert sorted(t.where(t.a <= threshold)["id"][:].tolist()) == [0] + + +def test_dictionary_inequality_no_longer_returns_nulls(): + """A dictionary null is a reserved *code*, so ``!= 'Uber'`` used to be true + for it -- the code differs from every value's. It is unknown instead, and + ``== None`` stays the way to ask for the nulls.""" + Row = dataclasses.make_dataclass( + "R", + [ + ("id", int, blosc2.field(blosc2.int64())), + ("v", str, blosc2.field(blosc2.dictionary(nullable=True))), + ], + ) + t = blosc2.CTable(Row, new_data=[(0, "Uber"), (1, "Lyft"), (2, None), (3, "Via")]) + assert sorted(t.where(t.v != "Uber")["id"][:].tolist()) == [1, 3] + assert sorted(t.where(t.v == "Uber")["id"][:].tolist()) == [0] + assert sorted(t.where(~(t.v == "Uber"))["id"][:].tolist()) == [1, 3] + assert sorted(t.where(t.v == None)["id"][:].tolist()) == [2] # noqa: E711 + assert (t.v != "Uber").is_null().tolist() == [False, False, True, False] + + +def test_utf8_negation_drops_nulls(): + Row = dataclasses.make_dataclass( + "R", + [ + ("id", int, blosc2.field(blosc2.int64())), + ("s", str, blosc2.field(blosc2.utf8(nullable=True))), + ], + ) + t = blosc2.CTable(Row, new_data=[(0, "aa"), (1, "zz"), (2, None), (3, "")]) + assert sorted(t.where(~(t.s == "zz"))["id"][:].tolist()) == [0, 3] + assert (t.s == "zz").is_null().tolist() == [False, False, True, False] + + +# --------------------------------------------------------------------------- +# Differential: string form, operator form, oracle -- indexed and not +# --------------------------------------------------------------------------- + + +def _oracle_table(storage, n=2000, seed=7): + rng = np.random.default_rng(seed) + a = rng.integers(0, 100, n) + b = rng.integers(0, 100, n) + a_null = rng.random(n) < 0.15 + b_null = rng.random(n) < 0.15 + spec, null_cell = nullable(blosc2.int64, storage, -9) + Row = dataclasses.make_dataclass( + "R", + [ + ("id", int, blosc2.field(blosc2.int64())), + ("a", int, blosc2.field(spec)), + ("b", int, blosc2.field(spec)), + ], + ) + rows = [ + (i, null_cell if a_null[i] else int(a[i]), null_cell if b_null[i] else int(b[i])) for i in range(n) + ] + return blosc2.CTable(Row, new_data=rows), a, b, a_null, b_null + + +def _kleene_pairs(a, b, a_null, b_null): + """``(true, unknown)`` NumPy channels for the leaves used below.""" + + def leaf(values, nulls, op, x): + return (op(values, x) & ~nulls, nulls) + + return { + "a > 50": leaf(a, a_null, np.greater, 50), + "b < 20": leaf(b, b_null, np.less, 20), + "a != 50": leaf(a, a_null, np.not_equal, 50), + } + + +def _np_and(x, y): + (ta, ua), (tb, ub) = x, y + return (ta & tb, (ua & (tb | ub)) | (ub & (ta | ua))) + + +def _np_or(x, y): + (ta, ua), (tb, ub) = x, y + return (ta | tb, (ua & ~tb) | (ub & ~ta)) + + +def _np_not(x): + t, u = x + return (~t & ~u, u) + + +SHAPES = [ + ("a > 50", lambda t: t.a > 50, lambda L: L["a > 50"]), + ("a != 50", lambda t: t.a != 50, lambda L: L["a != 50"]), + ("~(a > 50)", lambda t: ~(t.a > 50), lambda L: _np_not(L["a > 50"])), + ("(a > 50) & (b < 20)", lambda t: (t.a > 50) & (t.b < 20), lambda L: _np_and(L["a > 50"], L["b < 20"])), + ("(a > 50) | (b < 20)", lambda t: (t.a > 50) | (t.b < 20), lambda L: _np_or(L["a > 50"], L["b < 20"])), + ( + "~((a > 50) & (b < 20))", + lambda t: ~((t.a > 50) & (t.b < 20)), + lambda L: _np_not(_np_and(L["a > 50"], L["b < 20"])), + ), + ( + "~((a > 50) | (b < 20))", + lambda t: ~((t.a > 50) | (t.b < 20)), + lambda L: _np_not(_np_or(L["a > 50"], L["b < 20"])), + ), + ( + "~((a > 50) & ~(b < 20))", + lambda t: ~((t.a > 50) & ~(t.b < 20)), + lambda L: _np_not(_np_and(L["a > 50"], _np_not(L["b < 20"]))), + ), +] + + +@pytest.mark.parametrize("storage", STORAGES) +@pytest.mark.parametrize("indexed", [False, True], ids=["scan", "indexed"]) +@pytest.mark.parametrize(("expr", "build", "oracle"), SHAPES, ids=[s[0] for s in SHAPES]) +def test_matches_a_numpy_kleene_oracle(storage, indexed, expr, build, oracle): + """Four ways to the same answer: the string form, the operator form, and a + NumPy transcription of the truth tables -- with and without an index. + + An ordered index never evaluates the predicate, so an expression being + null-aware cannot fix its result; the index paths have their own null + handling, and this is what keeps the two in step. + """ + t, a, b, a_null, b_null = _oracle_table(storage) + if indexed: + t.create_index("a") + t.create_index("b") + want_true, want_unknown = oracle(_kleene_pairs(a, b, a_null, b_null)) + expected = sorted(np.flatnonzero(want_true).tolist()) + + assert sorted(t.where(expr)["id"][:].tolist()) == expected + pred = build(t) + assert sorted(t.where(pred)["id"][:].tolist()) == expected + np.testing.assert_array_equal(pred.is_null(), want_unknown) + + +# --------------------------------------------------------------------------- +# Consumers other than where() +# --------------------------------------------------------------------------- + + +def test_getitem_filter_matches_where(): + t = int_pair_table("mask") + p = ~(t.a == 1) + assert t[p]["id"][:].tolist() == t.where(p)["id"][:].tolist() + + +def test_aggregate_where_takes_a_three_valued_predicate(): + """``where=`` is a row filter, so it collapses like ``where()`` does: the + unknown rows are not counted.""" + t = int_pair_table("mask") + p = ~(t.a == 1) + kept = t.where(p) + assert t.id.sum(where=p) == kept.id.sum() + assert t.id.max(where=p) == kept.id.max() + + +def test_assign_stores_the_collapsed_predicate(): + """A computed column is an array of values with nowhere to keep a third + one, so the unknown rows land as False -- the same collapse ``where()`` + makes, stated where it happens. + + Sentinel storage, because a *mask* column's predicate reads its sidecar, + which is not a stored column of the table -- a computed column cannot + reference it. That limitation predates three-valued logic (the collapsed + predicate carried the same operand) and is unrelated to it. + """ + t = int_pair_table("sentinel") + flagged = t.assign(flag=~(t.a == 1)) + assert flagged.flag[:].tolist() == [k_not(x) is TRUE for x, _ in PAIRS] + + +def test_sorted_view_reports_unknown_rows_in_view_order(): + """``is_null()`` is logical, so it follows the view the predicate was built + on -- not the physical order the channels live in.""" + t = int_pair_table("mask") + view = t.sort_by("id", ascending=False) + p = view.a == 1 + assert p.is_null().tolist() == [x is UNKNOWN for x, _ in reversed(PAIRS)] diff --git a/tests/ctable/test_null_aware_indexes.py b/tests/ctable/test_null_aware_indexes.py index 7fd96959d..6ef62d5d1 100644 --- a/tests/ctable/test_null_aware_indexes.py +++ b/tests/ctable/test_null_aware_indexes.py @@ -348,11 +348,21 @@ def test_a_row_null_in_one_branch_still_matches_the_other(tmp_path): def test_a_pipe_inside_a_string_literal_is_not_an_or(): - """The OR test decides whether a nullable column keeps its index, so it + """The test decides whether a nullable column keeps its index, so it parses rather than searching for a character that string data may contain.""" - from blosc2.ctable_indexing import _expression_has_or + from blosc2.ctable_indexing import _expression_defeats_global_null_filter - assert _expression_has_or("(a > 1) | (b < 2)") - assert _expression_has_or("a > 1 or b < 2") - assert not _expression_has_or("name == 'a|b'") - assert not _expression_has_or("(a > 1) & (b < 2)") + assert _expression_defeats_global_null_filter("(a > 1) | (b < 2)") + assert _expression_defeats_global_null_filter("a > 1 or b < 2") + assert not _expression_defeats_global_null_filter("name == 'a|b'") + assert not _expression_defeats_global_null_filter("(a > 1) & (b < 2)") + + +def test_a_negation_also_defeats_the_global_null_filter(): + """Kleene negation is the second shape a global filter cannot express: + ``~(unknown & false)`` is true, so the null row it would drop qualifies.""" + from blosc2.ctable_indexing import _expression_defeats_global_null_filter + + assert _expression_defeats_global_null_filter("~(a > 1)") + assert _expression_defeats_global_null_filter("not (a > 1)") + assert _expression_defeats_global_null_filter("~((a > 1) & (b < 2))") diff --git a/tests/ctable/test_null_expressions.py b/tests/ctable/test_null_expressions.py index 29ab6b88c..3aa2999e8 100644 --- a/tests/ctable/test_null_expressions.py +++ b/tests/ctable/test_null_expressions.py @@ -124,16 +124,29 @@ def test_timestamp_comparisons_exclude_nulls(): assert t[t.ts > 1500]["id"][:].tolist() == [3] -def test_inverted_comparison_selects_null_rows(): - """Documented consequence of SQL False-semantics (not Kleene): a null - compares False, so negating a comparison *selects* null rows. The - escape hatch is the complementary comparison, which never matches - nulls.""" +def test_inverted_comparison_drops_null_rows(): + """Negation is Kleene, not bitwise: ``~unknown`` is unknown, so a null row + is no more a match for ``~(score > 0)`` than it was for ``score > 0``. + + This is what the two spellings below being *equal* means -- before + three-valued logic the negation collapsed the null to False first and then + inverted it, so it selected exactly the rows it should have excluded.""" t = CTable(IntRow, new_data=[(1, 10, 0), (2, NULL_I64, 0), (3, -20, 0)]) - assert t[~(t.score > 0)]["id"][:].tolist() == [2, 3] + assert t[~(t.score > 0)]["id"][:].tolist() == [3] assert t[t.score <= 0]["id"][:].tolist() == [3] +def test_negation_keeps_the_null_recoverable(): + """The unknown rows are not lost, just not matches: ``fillna(True)`` is the + other reading, and ``is_null()`` names them.""" + t = CTable(IntRow, new_data=[(1, 10, 0), (2, NULL_I64, 0), (3, -20, 0)]) + negated = ~(t.score > 0) + assert negated.is_null().tolist() == [False, True, False] + assert negated.null_count() == 1 + assert t[negated.fillna(True)]["id"][:].tolist() == [2, 3] + assert t[negated.fillna(False)]["id"][:].tolist() == [3] + + # =========================================================================== # Arithmetic: null propagates, output promoted to float64/NaN # =========================================================================== diff --git a/tests/ctable/test_null_predicate_rewrite.py b/tests/ctable/test_null_predicate_rewrite.py index 20761ba7d..ee376db48 100644 --- a/tests/ctable/test_null_predicate_rewrite.py +++ b/tests/ctable/test_null_predicate_rewrite.py @@ -15,6 +15,11 @@ Per *leaf*, not once over the whole expression: a global ``result & valid_a`` would drop a row that is null in ``a`` but matches the other branch of ``(a > 10) | (b == 0)``, which SQL says qualifies. + +Under a negation a guard is not enough in either position, and the rewrite +switches to three-valued logic there; ``tests/ctable/test_kleene_logic.py`` +covers that side, and the two negation tests here pin the string form against +the operator one. """ from __future__ import annotations @@ -179,42 +184,32 @@ def test_string_form_agrees_with_operator_form(label, spec, annotation, values, ) -def test_negation_over_and_corner_is_conservative(): - """The one known corner where the two query forms disagree -- pinned. +def test_negation_over_and_corner_agrees_with_sql(): + """The corner the two query forms used to disagree on -- now both exact. ``~((a > 10) & (b == 999))`` with ``a`` null and the second term False: SQL collapses ``NULL AND FALSE`` to ``FALSE``, so the negation is True and - the row qualifies. The string form guards at the negation point - (``... & (a != -1)``) and drops it. That is by design -- pushing validity - to the negation point is exact for every simpler form, and errs only by - *dropping*: a null row is never wrongly returned (see - plans/mask-based-nulls.md, "Negation caveat, resolved"). - - The operator form returns the SQL answer here, but not by reasoning in - three values: ``_null_aware_compare`` collapses null to False at the leaf, - so ``~`` sees plain booleans -- which is also why it leaks nulls on a bare - negation (the xfail below). If either assertion starts failing, the two - forms moved; make sure they moved *toward* SQL, together. + the row qualifies. Getting that right needs three values, because it + hinges on ``unknown & false`` being *false* rather than unknown -- no + amount of conjoining a validity guard onto the negation can express it + (the string form dropped the row until Kleene logic landed). """ t = _table(blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1) - got = sorted(t.where("~((a > 10) & (b == 999))")["b"][:].tolist()) - assert got == [0, 1, 3] # null row dropped; SQL would keep it - - got = sorted(t.where(~((t["a"] > 10) & (t["b"] == 999)))["b"][:].tolist()) - assert got == [0, 1, 2, 3] # SQL-exact + assert sorted(t.where("~((a > 10) & (b == 999))")["b"][:].tolist()) == [0, 1, 2, 3] + assert sorted(t.where(~((t["a"] > 10) & (t["b"] == 999)))["b"][:].tolist()) == [0, 1, 2, 3] -@pytest.mark.xfail( - strict=True, - reason="operator-form negation is not null-aware: `a > 10` collapses null to " - "False at the leaf, so ~ turns the null row into a match. The string form " - "gets this right via the negation-point guard. Fixing it needs the " - "comparison result to carry its null predicate through ~.", -) def test_operator_form_negation_drops_nulls(): + """``~(a > 10)`` is unknown for a null ``a``, so the row is not a match. + + Was a strict xfail: the comparison collapsed the null to False at the leaf + and ``~`` inverted it into a match. Fixed at the source -- the comparison + now carries its null predicate through ``~`` (:class:`NullableBoolExpr`). + """ t = _table(blosc2.int64(null_value=-1), int, [1, 20, -1, 30], -1) assert sorted(t.where(~(t["a"] > 10))["b"][:].tolist()) == [0] + assert sorted(t.where("~(a > 10)")["b"][:].tolist()) == [0] def test_comparing_against_the_sentinel_matches_nothing(): From 2115044a0293de83e09af40c537357d59f687d49 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 09:49:32 +0200 Subject: [PATCH 26/47] Fix the two defects a post-landing review turned up Both are in code the mask-based-nulls branch added, and neither was reachable by the suites it shipped with. **One value assigned to many rows crashed a mask column.** col[0:2] = 7 -- and the same write through a boolean mask or an index list -- raised TypeError: iteration over a 0-d array, because coerce_batch went looking for nulls *inside* a value that was a single cell rather than a batch, and np.asarray(7) is 0-d. Scalar broadcast has worked for fixed-width columns since long before this work, so mask storage becoming the default silently took it away from every plain nullable column. The suites missed it because every __setitem__ test assigns a list. coerce_batch now recognizes a broadcast cell first, through a new is_broadcast_cell. A non-null cell passes through untouched, so the value write broadcasts exactly as it does for a non-mask column; None returns the fill plus an all-invalid array, which makes col[0:2] = None the broadcast spelling of "null these rows" -- new capability only mask storage can offer. Two boundaries drawn from what sentinel storage already does rather than from what would be nice. utf8 keeps raising ValueError: Length mismatch for a broadcast *value*, because a varlen column has never taken one and a sentinel utf8 column says the same thing today; only the None form is made to work there, by spelling the fill out n times. And a 1-d array handed to a fixed-shape ndarray column stays a batch: one item and a batch of items differ by dimension alone, so guessing wrong writes bad data instead of raising. **convert_nulls(to="sentinel") refused over a value in a deleted row.** The collision check scanned physical slots, so a proposed sentinel present only in an already-deleted row -- unreadable through any API, and dropped by the next compact() -- blocked the conversion. That contradicts the check's own reason for existing, which is that reserving a value must not relabel a *live* row as null. It now intersects with _valid_rows, and reports the logical row number the caller can index rather than a physical slot, which on a table with holes named a different row. Both are skipped on a dense table, where the physical position already is the logical one, so the common case reads no extra array. Verified by mutation, each half separately: reverting the broadcast branch fails 17 of the new tests, reverting the _valid_rows intersection fails the dead-slot test, and reverting the logical-row conversion fails the row-numbering test. tests/ctable: 2648 passed. Two adjacent rough edges left alone deliberately, both predating this branch and both identical under either storage: a timestamp column rejects a broadcast scalar from its own coercion path, and assign(scalar) reports len() of unsized object instead of the ValueError it documents. Co-Authored-By: Claude Fable 5 --- RELEASE_NOTES.md | 12 ++++++ plans/mask-based-nulls.md | 63 ++++++++++++++++++++++++++++ src/blosc2/ctable.py | 17 +++++++- src/blosc2/ctable_nulls.py | 49 +++++++++++++++++++++- tests/ctable/test_null_mask_api.py | 64 +++++++++++++++++++++++++++++ tests/ctable/test_null_migration.py | 32 +++++++++++++++ 6 files changed, 233 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index df0f7597b..8cd63626f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -173,6 +173,18 @@ and has its own spelling for nulls (`None` among the values). were data, and `from_csv` had nothing to put in an empty field and raised. Both go through the sidecar now: an empty CSV field is a null in either direction. Sentinel columns keep writing their sentinel, unchanged. +- **Assigning one value to many rows raised on a mask-storage column.** + `t.a[0:2] = 7` — and the same write through a boolean mask or an index list — + failed with `TypeError: iteration over a 0-d array`, because the write path + looked for nulls *inside* a value that was a single cell rather than a batch. + Scalar broadcast has always worked for fixed-width columns and works again + here. `t.a[0:2] = None` broadcasts too, making every selected row null. +- **`convert_nulls(to="sentinel")` refused over a value in a deleted row.** The + collision check scanned physical slots, so a proposed sentinel present only in + a row already deleted — unreadable, and dropped by the next `compact()` — + blocked the conversion. Only live rows are consulted now, and the row named in + the refusal is the logical one the caller can index rather than a physical + slot. ## Changes from 4.10.0 to 4.10.1 diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 57d927b75..de39b5482 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1037,3 +1037,66 @@ channel to one more builder rather than to teach a new subsystem about nulls. `_exclude_null_positions` go away entirely, put `OR` on the exact-position paths, and give `_sorted_slice_positions` (which bails since Phase 7) its window read back. It needs the sort key and the external-merge builder to carry validity, in every index kind. + +## Post-landing review findings (2026-08-10) + +A review pass over the whole branch, cut short before the multi-angle `/code-review` sweep could +finish (stopped for credit reasons — only its line-by-line angle reported, and re-running the full +sweep later is worth it). Two confirmed defects, both reproduced against this tree, **both fixed +2026-08-10** — see the as-built note after each: + +1. **Scalar broadcast writes crash on a mask column — every key shape.** `col[0:2] = 7`, + `col[bool_mask] = 5`, `col[[1]] = 9` and `col[0:2] = None` all raise + `TypeError: iteration over a 0-d array`; the same writes on a sentinel column broadcast fine, + and list-valued writes (`col[0:2] = [None, 5]`) work on both storages. The mask write path hands + the scalar to batch coercion, where `np.asarray(scalar)` is 0-d and the null-detection + iteration dies on it. Since mask is now the **default** storage, this breaks a plain + `t.a[i:j] = value` on any freshly created nullable column. Fix by broadcasting the scalar to the + key's length before `coerce_batch` (or teaching `split_batch_validity` the 0-d case), and note + the test gap that hid it: the `__setitem__` suites always assign list values, never a bare + scalar. *(Found by the review's line-by-line angle; verified by hand.)* + + > **Fixed 2026-08-10.** `NullChannel.coerce_batch` now recognizes a **broadcast cell** before it + > looks for nulls inside the value, via a new `is_broadcast_cell` in `ctable_nulls.py`. A + > non-NA cell is passed through untouched, so the value write broadcasts exactly as it does for + > a non-mask column; `None` returns the fill plus an all-invalid array, which makes + > `t.a[i:j] = None` the broadcast spelling of "null these rows" — new capability that only mask + > storage can offer. Two details decided by looking at what sentinel storage already does rather + > than by inventing behavior: **utf8 keeps raising `ValueError: Length mismatch`** for a + > broadcast value, because a varlen column has never accepted one and a sentinel utf8 column + > says the same thing today (only the `None` form is made to work there, by spelling the fill + > out *n* times); and a **1-d array handed to a fixed-shape ndarray column stays a batch**, + > since one item and a batch of items differ by dimension alone and misreading one as the other + > would write the wrong thing rather than raise. Pinned by 17 tests in `test_null_mask_api.py`, + > including one asserting the two storages agree cell for cell on a broadcast write; all 17 fail + > when the branch is reverted. + > + > Not fixed, and out of scope: a **timestamp** column rejects a broadcast scalar under *both* + > storages (`IndexError: too many indices for array: array is 0-dimensional`), which predates + > this work and lives in the timestamp coercion path, not the null one. + +2. **`convert_nulls(to="sentinel")` falsely refuses on a value that lives only in a dead slot.** + `_convert_sentinel_clash` (`ctable.py:11158`) intersects its hit mask with the validity sidecar + but not with `_valid_rows`, so a proposed sentinel present only in a *deleted* row's physical + slot — unobservable, and removed for good by the next `compact()` — refuses the conversion. + Repro: mask `int64` `[1, -9, 3, None]`, `delete(1)`, then `convert_nulls(to="sentinel", + null_value=-9)` → refused, though no live row holds `-9`. The message also reports the + *physical* slot as "row 1", which on a holed table is not the row number the user sees. The fix + is one more `&` against `_valid_rows` in the clash scan, plus a logical row number in the + message. Low severity — needs deletions plus a collision confined to dead slots — but the + refusal contradicts the check's own rationale, which is about *relabelling live rows*. + + > **Fixed 2026-08-10.** Both halves, as described: the scan intersects with `_valid_rows`, and + > the reported row is converted to a logical one with `count_nonzero(live[:phys])`. Both are + > skipped on a dense table (`_resolve_last_pos() == _n_rows`), where every physical slot is a + > live row and its position already *is* the logical one, so the common case reads no extra + > array. Pinned by two tests in `test_null_migration.py` — the dead-slot value that must not + > block, and the live value that must still block *and* name a row the caller can index — which + > fail independently when each half is reverted. + +Checked and clean, for the record: the full `tests/ctable` suite (2629 tests) passes on the +branch; the `extend()` crash-safety ordering carries its do-not-reorder comment; the conditional +version-3 gate matches the design; Arrow export of sorted views, filtered views, and holed tables +places nulls on the right rows; overwrite-replaces-validity holds across slice, boolean-mask and +fancy keys (with list values — see finding 1); and sentinel→mask conversion of a table with +deleted rows is correct, including a deleted row that held the sentinel. diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 915a24578..3e7fff291 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -11156,22 +11156,35 @@ def _sentinel_for_conversion(self, name: str, null_value: Any): return null_value def _convert_sentinel_clash(self, col: Column, spec, null_value) -> int | None: - """The first non-null row already holding *null_value*, or ``None``. + """The first live non-null row already holding *null_value*, or ``None``. A NaN sentinel never clashes: under mask storage NaN is a value, but it is one the sentinel model has always spelled "null", so folding the two together is the documented semantic change rather than data loss. + + Only **live** rows are consulted, and the answer is a *logical* row + number. A deleted row's slot keeps whatever it held until the next + ``compact()``, but no API can read it and the next compaction drops it, + so a value confined to dead slots is not data the conversion could + relabel -- which is the whole reason this check exists. """ if is_nan_sentinel(null_value): return None valid_arr = self._null_mask(col._col_name) + live = self._valid_rows + # Dense: no deletions, so every physical slot is a live row and its + # position is already the logical one. + dense = self._resolve_last_pos() == self._n_rows item_ndim = col.item_ndim if col.is_ndarray else 0 for start, stop, raw in self._iter_convert_spans(col._col_name): hit = sentinel_mask(raw, null_value, item_ndim=item_ndim) if valid_arr is not None: hit &= np.asarray(valid_arr[start:stop], dtype=bool) + if not dense: + hit &= np.asarray(live[start:stop], dtype=bool) if hit.any(): - return start + int(np.flatnonzero(hit)[0]) + phys = start + int(np.flatnonzero(hit)[0]) + return phys if dense else int(np.count_nonzero(np.asarray(live[:phys]))) return None def _iter_convert_spans(self, name: str): diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index dc0009734..99acf45ca 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -59,6 +59,7 @@ "NullChannel", "fill_item_for", "fill_value_for", + "is_broadcast_cell", "is_na_marker", "is_nan_sentinel", "is_null_value", @@ -138,6 +139,35 @@ def fill_item_for(spec): return base +def is_broadcast_cell(values) -> builtin_bool: + """True when *values* is one cell to spread over every selected row. + + ``col[0:2] = 7`` has always broadcast a scalar across the selection, the + same way assigning to a NumPy slice does, and ``col[0:2] = None`` is the + mask-storage spelling of the same thing for a null. Batch coercion has to + recognize both before it starts looking for nulls *inside* the value: + ``np.asarray(7)`` is 0-d, and iterating it raises rather than yielding one + element. + + Only unambiguous single cells qualify. A list, tuple or array is a batch, + including a 1-d array handed to a fixed-shape ndarray column -- there one + item and a batch of items differ by dimension alone, and misreading a batch + as a cell would write the wrong thing rather than raise. + """ + if is_na_marker(values): + return True + if isinstance(values, np.ndarray): + return values.ndim == 0 + if isinstance(values, (list, tuple, blosc2.NDArray)): + return False + # str/bytes are sized but are single cells; the rest are the scalar types a + # column can hold -- NumPy scalars (np.generic covers datetime64) and the + # Python ones, including datetime for a timestamp column. + return isinstance(values, (str, builtin_bytes, builtin_bool, int, float, complex, np.generic)) or ( + not hasattr(values, "__len__") and not hasattr(values, "__iter__") + ) + + def split_batch_validity(values, fill): """Split an incoming batch into ``(storage_values, valid)``. @@ -854,11 +884,26 @@ def coerce_batch(self, values, n: int): Thin wrapper over :func:`split_batch_validity`, which carries the null detection rules and the reasoning behind them; all this adds is the - column's own fill, and the short-circuit for a column whose nulls do - not live in a sidecar at all. + column's own fill, the short-circuit for a column whose nulls do not + live in a sidecar at all, and the single-cell case below. + + A **broadcast cell** (:func:`is_broadcast_cell`) is one value for all + *n* rows rather than a batch of them. ``None`` there makes every + selected row null; anything else is passed through untouched, so the + value write broadcasts exactly as it does for a non-mask column and a + kind that never accepted a broadcast keeps saying so in its own words. """ if self.kind != NULL_MASK: return values, None + if is_broadcast_cell(values): + if not is_na_marker(values): + return values, None + fill = self.fill_value + if self._col.is_varlen_scalar or self._col.is_list: + # These are written cell by cell, so they need the fill spelled + # out n times where the fixed-width kinds broadcast one. + fill = [fill] * n + return fill, np.zeros(n, dtype=np.bool_) return split_batch_validity(values, self.fill_value) # ------------------------------------------------------------------ diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index 836f6379b..a5808ffb4 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -302,6 +302,70 @@ def test_setitem_index_list_replaces_validity(): assert t["a"].is_null().tolist() == [False, False, False, True] +# --------------------------------------------------------------------------- +# One value for many rows. Assigning a scalar to a selection has always +# broadcast it, the way NumPy does, and mask storage must not take that away -- +# it is the default now, so ``t.a[i:j] = value`` on a plain nullable column +# goes through here. ``None`` broadcasts the same way, which is the mask +# spelling of "make these rows null". +# --------------------------------------------------------------------------- + + +def keys_for(t): + """The three multi-row key forms, each selecting rows 0 and 1.""" + return [slice(0, 2), np.array([True, True, False]), [0, 1]] + + +@pytest.mark.parametrize("key_index", [0, 1, 2], ids=["slice", "bool_mask", "index_list"]) +def test_setitem_broadcasts_a_scalar_over_every_key_form(key_index): + t = simple([1, 2, 3]) + t["a"][keys_for(t)[key_index]] = 7 + assert t["a"][:].tolist() == [7, 7, 3] + assert t["a"].is_null().tolist() == [False, False, False] + + +@pytest.mark.parametrize("key_index", [0, 1, 2], ids=["slice", "bool_mask", "index_list"]) +def test_setitem_broadcasts_none_over_every_key_form(key_index): + t = simple([1, 2, 3]) + t["a"][keys_for(t)[key_index]] = None + assert t["a"].is_null().tolist() == [True, True, False] + + +@pytest.mark.parametrize(("label", "spec", "value"), WRITEABLE_SPECS) +def test_setitem_broadcasts_one_cell_for_every_kind(label, spec, value): + t = table([(value,), (value,), (value,)], a=spec) + if label == "utf8": + # A varlen column has never accepted a broadcast value -- it says so in + # its own words, and a sentinel utf8 column says exactly the same. + with pytest.raises(ValueError, match="Length mismatch"): + t["a"][0:2] = value + else: + t["a"][0:2] = value + assert t["a"].is_null().tolist() == [False, False, False] + t["a"][0:2] = None + assert t["a"].is_null().tolist() == [True, True, False] + + +def test_broadcasting_a_scalar_matches_sentinel_storage(): + """The two storages agree on a broadcast write, value and validity alike.""" + mask = simple([1, 2, 3]) + sentinel = simple([1, 2, 3], spec=blosc2.int64(null_storage="sentinel", null_value=-9)) + mask["a"][0:2] = 7 + sentinel["a"][0:2] = 7 + assert mask["a"][:].tolist() == sentinel["a"][:].tolist() + assert mask["a"].is_null().tolist() == sentinel["a"].is_null().tolist() + + +def test_broadcasting_none_clears_a_previously_written_value(): + """A broadcast null replaces validity rather than merging with it.""" + t = simple([1, None, 3]) + t["a"][0:3] = None + assert t["a"].null_count() == 3 + t["a"][0:3] = 5 + assert t["a"].null_count() == 0 + assert t["a"][:].tolist() == [5, 5, 5] + + # --------------------------------------------------------------------------- # The null API # --------------------------------------------------------------------------- diff --git a/tests/ctable/test_null_migration.py b/tests/ctable/test_null_migration.py index cef2612af..938447d2d 100644 --- a/tests/ctable/test_null_migration.py +++ b/tests/ctable/test_null_migration.py @@ -230,6 +230,38 @@ def test_a_nan_already_present_does_not_block_the_nan_sentinel(): assert converted["a"].is_null().tolist() == [False, True, True] +def test_a_value_only_in_a_deleted_row_does_not_block_the_conversion(): + """A dead slot holds no data the conversion could relabel. + + Deleting a row leaves its physical slot untouched until the next + ``compact()``, but nothing can read it and compaction drops it, so refusing + the conversion over it would block a migration on a value that no longer + exists. The check is about relabelling **live** rows as null. + """ + t = one_col([1, -9, 3, None], blosc2.int64(null_storage="mask")) + t.delete(1) + assert as_list(t["a"]) == [1, 3, None] + + converted = t.convert_nulls("a", to="sentinel", null_value=-9) + assert converted["a"].null_storage == "sentinel" + assert as_list(converted["a"]) == [1, 3, None] + + +def test_a_live_value_still_blocks_it_and_the_row_is_the_logical_one(): + """The refusal survives deletions, and reports a row the caller can index. + + A physical slot number would name a different row than ``col[row]`` on a + table with holes, sending the reader to a value that is not the offending + one. + """ + t = one_col([1, 2, -9, None], blosc2.int64(null_storage="mask")) + t.delete(0) + assert as_list(t["a"]) == [2, -9, None] # the -9 is logical row 1, physical slot 2 + + with pytest.raises(ValueError, match="already contains -9 at row 1"): + t.convert_nulls("a", to="sentinel", null_value=-9) + + def test_the_refusal_leaves_the_column_alone(): t = one_col([*range(-128, 128), None], blosc2.int8(null_storage="mask"), capacity=300) with pytest.raises(ValueError): From 6e719c841433c3448d55529957e1fc60ee5d5a7d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 15:05:18 +0200 Subject: [PATCH 27/47] Let the Kleene suite run on NumPy 1.x too blosc2.utf8() raises without numpy.dtypes.StringDType, and test_kleene_logic.py called it in two places the earlier compat pass did not cover -- it landed after that pass. Three failures on the ubuntu NumPy 1.26 job. The parametrize entry passes blosc2.utf8 as a *factory*, uncalled, so collection survived and it was the test body that raised; that is why this showed up as failures rather than the collection errors the other suites hit. Both sites take the established guard from tests/ctable/utf8_compat.py: the kind case becomes a pytest.param carrying needs_utf8, and the utf8-only test takes the mark directly. Verified the way the earlier pass was, by deleting numpy.dtypes.StringDType before collection: 73 passed, 3 skipped, against 76 passed with it present -- the same three tests CI reported. Co-Authored-By: Claude Fable 5 --- tests/ctable/test_kleene_logic.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/ctable/test_kleene_logic.py b/tests/ctable/test_kleene_logic.py index cd43af5dd..2aa5df5aa 100644 --- a/tests/ctable/test_kleene_logic.py +++ b/tests/ctable/test_kleene_logic.py @@ -33,6 +33,7 @@ import numpy as np import pytest +from utf8_compat import needs_utf8 import blosc2 from blosc2.ctable import NullableBoolExpr @@ -316,7 +317,17 @@ def test_the_predicate_is_a_lazy_expression(): b"mm", b"ZZZZZZZZ", ), - ("utf8", blosc2.utf8, str, ["aa", "zz", None, "yy"], "mm", "__BLOSC2_NULL__"), + # The factory is passed uncalled, so collection survives on NumPy 1.x and it + # is the test body that would raise; the mark is what keeps it from running. + pytest.param( + "utf8", + blosc2.utf8, + str, + ["aa", "zz", None, "yy"], + "mm", + "__BLOSC2_NULL__", + marks=needs_utf8, + ), ] @@ -360,6 +371,7 @@ def test_dictionary_inequality_no_longer_returns_nulls(): assert (t.v != "Uber").is_null().tolist() == [False, False, True, False] +@needs_utf8 def test_utf8_negation_drops_nulls(): Row = dataclasses.make_dataclass( "R", From beabc9c81937d8f440662964847d5d9ae3254060 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 15:09:33 +0200 Subject: [PATCH 28/47] Stop convert_nulls() flattening a nested column _detach_schema deep-copies the specs so an in-place conversion cannot relabel its source's columns, and it rebuilt CompiledSchema from three fields to do it. Everything else the schema carries was dropped in the process, so a struct column came back flattened -- trip exported as trip.lon and trip.lat. _convert_nulls_inplace detaches before it looks at whether there is any work, so a table with no nullable column at all went through it: t.convert_nulls() as a pure no-op was enough. With inplace=True on a persistent table _retype_converted_column then wrote the damaged schema to disk. Two things had to move, and the second is the one that actually flattened it. dataclasses.replace keeps metadata (metadata["nested"] drives the reconstruction and is serialized with the schema) and validator_model. But columns_by_name is not {c.name for c in columns}: it also carries the logical parent of a nested group, a StructSpec entry for trip beside its leaves, which _export_arrow_names looks up precisely because it is absent from col_names. Rebuilding the dict from columns dropped those parents. Every key is preserved now, reusing the new column object where there is one. Found while verifying and deliberately left alone: reopening a persisted nested table flattens it with no conversion involved at all, because schema_from_dict does not rebuild the struct parent either. Confirmed identical before and after this change, so it predates this branch and belongs to the nested-metadata work. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 29 ++++++++++++--- tests/ctable/test_null_migration.py | 55 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 3e7fff291..3efd488c0 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -11209,13 +11209,32 @@ def _detach_schema(self) -> None: thing that does not. Conversion rewrites specs **in place**, and without this a converted copy would reach back and relabel its source's columns too, leaving that table reporting a storage its data does not use. + + Only the specs are replaced, and *nothing else about the schema moves*. + Two things made that harder than it looks, and both flattened a struct + column -- ``trip`` exported as ``trip.lon``/``trip.lat`` -- when a + conversion, or merely a no-op request for one, ran over a nested table: + + - ``dataclasses.replace`` rather than a fresh ``CompiledSchema``, which + would drop ``metadata`` (``metadata["nested"]`` drives the + reconstruction and is serialized with the schema) and + ``validator_model``. + - ``columns_by_name`` is **not** ``{c.name for c in columns}``. It also + carries the logical parent of a nested group -- a ``StructSpec`` entry + for ``trip`` beside its ``trip.lon``/``trip.lat`` leaves -- which + ``_export_arrow_names`` looks up precisely because it is absent from + ``col_names``. Rebuilding the dict from ``columns`` silently dropped + those parents. + + An in-place conversion then persisted the damage through ``save_schema``. """ columns = [dataclasses.replace(c, spec=copy.deepcopy(c.spec)) for c in self._schema.columns] - self._schema = CompiledSchema( - row_cls=self._schema.row_cls, - columns=columns, - columns_by_name={c.name: c for c in columns}, - ) + detached = {c.name: c for c in columns} + columns_by_name = { + name: detached.get(name) or dataclasses.replace(cc, spec=copy.deepcopy(cc.spec)) + for name, cc in self._schema.columns_by_name.items() + } + self._schema = dataclasses.replace(self._schema, columns=columns, columns_by_name=columns_by_name) def _convert_nulls_inplace(self, targets: list[tuple[str, Any]], to: str) -> None: """Rewrite each target column's null channel, one column at a time. diff --git a/tests/ctable/test_null_migration.py b/tests/ctable/test_null_migration.py index 938447d2d..d23682248 100644 --- a/tests/ctable/test_null_migration.py +++ b/tests/ctable/test_null_migration.py @@ -582,6 +582,61 @@ def test_a_converted_column_round_trips_through_arrow(): assert arrow.column("a").to_pylist() == [True, None, False] +def nested_table(null_storage="mask"): + """A table with a struct column beside a nullable scalar one.""" + pa = pytest.importorskip("pyarrow") + return blosc2.CTable.from_arrow( + pa.table( + { + "a": pa.array([1, None, 3], pa.int64()), + "trip": pa.array([{"lon": 1.0, "lat": 2.0}] * 3), + } + ), + null_storage=null_storage, + ) + + +def test_converting_keeps_a_struct_column_nested(): + """Conversion must not reach outside the column it converts. + + ``_detach_schema`` deep-copies the specs so an in-place rewrite cannot + relabel the source table's columns, and it has to leave everything *else* + the schema carries alone: ``metadata["nested"]``, which drives the + reconstruction, and the logical ``trip`` parent in ``columns_by_name``, + which is absent from ``columns`` precisely because it is not a physical + column. + """ + t = nested_table() + assert t.to_arrow().schema.names == ["a", "trip"] + + converted = t.convert_nulls("a", to="sentinel", null_value=-1) + assert converted.to_arrow().schema.names == ["a", "trip"] + assert converted.to_arrow().column("a").to_pylist() == [1, None, 3] + assert converted.to_arrow().column("trip").to_pylist()[0] == {"lon": 1.0, "lat": 2.0} + + +def test_a_no_op_conversion_leaves_a_nested_table_alone(): + """The damaging shape: nothing to convert, so nothing may change. + + ``_convert_nulls_inplace`` detaches the schema before it looks at whether + there is any work, so a table with no nullable column at all went through + the same path -- and came back flattened. + """ + pa = pytest.importorskip("pyarrow") + t = blosc2.CTable.from_arrow( + pa.table({"id": pa.array([1, 2, 3], pa.int64()), "trip": pa.array([{"lon": 1.0, "lat": 2.0}] * 3)}) + ) + assert t.convert_nulls().to_arrow().schema.names == ["id", "trip"] + + +def test_converting_does_not_relabel_the_source_schema(): + """The guarantee _detach_schema exists for, still holding after the fix.""" + t = nested_table() + converted = t.convert_nulls("a", to="sentinel", null_value=-1) + assert converted["a"].null_storage == "sentinel" + assert t["a"].null_storage == "mask" + + def test_extending_a_converted_column_accepts_none(): t = one_col([5, -1], blosc2.int64(nullable=True, null_value=-1)) converted = t.convert_nulls("a", to="mask") From d2c57db2e78d12c5b7dd0b53cd3894da6e8ef385 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 15:09:45 +0200 Subject: [PATCH 29/47] Reverse a descending sort by complement, not by negation The descending value key negated in int64, which fixed bool and the narrow signed dtypes but left the widest two wrong, both newly reachable because mask storage is what lets a nullable column hold the value each case turns on. int64's own minimum is a fixed point of negation -- -(-2**63) is -2**63 -- so that row sorted as if it were the largest: [5, INT64_MIN, 7, 1] came back as [INT64_MIN, 7, 5, 1]. That is exactly the bug the narrow dtypes were fixed for, one width up, and the comment above the line claimed it was handled. A uint64 above 2**63 was worse and was not reported at all: casting it to int64 to negate wrapped it negative, so [1, UINT64_MAX, UINT64_MAX - 1] sorted descending with the smallest value first. ~x is -x - 1 in two's complement, so it reverses the total order of every integer width exactly and has no fixed point to be caught by. It is also the same single pass; the rank inversion the string branch uses would have added an argsort. Ascending never negated and is untouched. Verified across int8/int32/int64/uint8/uint64/bool/float64 at their extremes, both directions, with the uint64 case checked against the pre-fix code to confirm it was already broken. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 20 ++++---- tests/ctable/test_null_mask_sort_groupby.py | 53 +++++++++++++++++++++ 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 3efd488c0..4bea0eed6 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -13717,14 +13717,18 @@ def _build_lex_keys( rank = np.argsort(np.argsort(raw, kind="stable"), kind="stable") lex_keys.append((n - 1 - rank).astype(np.intp)) elif raw.dtype.kind in "bui": - # Negate in int64, never in the column's own dtype. bool has - # no unary minus at all, and a narrow signed dtype wraps on - # its own minimum -- ``-(-128)`` is ``-128`` in int8 -- which - # silently leaves that row sorted as if it were the largest. - # Mask storage is what made both reachable in a *nullable* - # column: a sentinel had to reserve int8's -128, and a - # nullable bool was physically uint8. - lex_keys.append(-raw.astype(np.int64)) + # Reverse the order by complement, not by negation. ``~x`` + # is ``-x - 1`` in two's complement, so it reverses the total + # order of *every* integer width exactly, with no value that + # can overflow: int8's -128 maps to 127, int64's minimum to + # its maximum, and a uint64 above 2**63 stays put instead of + # wrapping negative. Negation has no such fixed point -- + # ``-x`` maps a dtype's minimum to itself, leaving that row + # sorted as if it were the largest -- and bool has no unary + # minus at all. Mask storage is what made these reachable in + # a *nullable* column: a sentinel had to reserve int8's -128 + # and int64's minimum, and a nullable bool was uint8. + lex_keys.append(np.bitwise_not(raw)) else: lex_keys.append(-raw) else: diff --git a/tests/ctable/test_null_mask_sort_groupby.py b/tests/ctable/test_null_mask_sort_groupby.py index 88a135be1..9d3c89b96 100644 --- a/tests/ctable/test_null_mask_sort_groupby.py +++ b/tests/ctable/test_null_mask_sort_groupby.py @@ -37,6 +37,10 @@ import blosc2 +INT64_MIN = int(np.iinfo(np.int64).min) +INT64_MAX = int(np.iinfo(np.int64).max) +UINT64_MAX = int(np.iinfo(np.uint64).max) + def annotation_for(spec): if isinstance(spec, (blosc2.schema.NDArraySpec, blosc2.schema.timestamp)): @@ -169,6 +173,55 @@ def test_descending_sort_of_a_plain_bool_column(): assert as_list(t.sort_by("a", ascending=False)["a"]) == [True, True, False] +@pytest.mark.parametrize( + ("spec", "values", "expected"), + [ + ( + blosc2.int64(null_storage="mask"), + [5, INT64_MIN, 7, None, 1], + [7, 5, 1, INT64_MIN, None], + ), + ( + blosc2.int64(null_storage="mask"), + [INT64_MAX, INT64_MIN, 0, None], + [INT64_MAX, 0, INT64_MIN, None], + ), + ( + blosc2.uint64(null_storage="mask"), + [1, UINT64_MAX, UINT64_MAX - 1, None], + [UINT64_MAX, UINT64_MAX - 1, 1, None], + ), + ], +) +def test_descending_sort_over_the_widest_integers(spec, values, expected): + """The two extremes narrowing to int64 could not express. + + The descending key used to negate in int64, which has a fixed point of its + own -- ``-(-2**63)`` is ``-2**63`` -- so the smallest ``int64`` sorted as if + it were the largest, exactly the bug the narrow dtypes were fixed for. A + ``uint64`` above ``2**63`` was worse: casting it to int64 to negate wrapped + it negative, so the two largest values sorted below ``1``. Complementing + instead of negating reverses every width exactly, having no fixed point. + + Mask storage is what makes these reachable in a nullable column, since a + sentinel had to reserve the very value each case turns on. + """ + assert as_list(one_col(values, spec).sort_by("a", ascending=False)["a"]) == expected + + +@pytest.mark.parametrize( + ("spec", "values"), + [ + (blosc2.int64(null_storage="mask"), [5, INT64_MIN, 7, None, 1]), + (blosc2.uint64(null_storage="mask"), [1, UINT64_MAX, UINT64_MAX - 1, None]), + ], +) +def test_ascending_sort_of_the_widest_integers_is_unchanged(spec, values): + """Ascending never negated, so it was always right; keep it that way.""" + live = sorted(v for v in values if v is not None) + assert as_list(one_col(values, spec).sort_by("a")["a"]) == [*live, None] + + # --------------------------------------------------------------------------- # The FULL-index sort path reads the sidecar, not the whole column # --------------------------------------------------------------------------- From 934ca99237b567fc7ee30adf3776f20c93015880 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 15:09:56 +0200 Subject: [PATCH 30/47] Read a mask column's nulls in the order its values arrive nonnull_chunks() zipped null_mask() against iter_chunks(), and the two walk the rows in different orders. iter_chunks steps through the validity array chunk by chunk and never consults a view's ordering, so it yields physically; null_mask() gathers through _resolve_live_positions(), so it answers in view order. On any ordered view the pairing was therefore off, dropping live values and letting the fill through as data. Every reduction routed through _nonnull_chunks was affected. On a sorted view of [10.0, None, 30.0, None, 50.0], sum() returned nan -- from a column whose nulls are not NaN -- and unique() returned [50., nan, 30.]: the fill escaping as a value while a real 10.0 disappeared. The sentinel path tests the values themselves (chunk != nv), which is order-independent, so it was correct on the same data and mask storage alone was wrong. The flags are now gathered at the live positions in physical order. Both streams read the same _valid_rows, which is what keeps a filtered view in step as well. The suites had no case where a view's order differed from storage order, which is why 2600 tests passed over it. Five now do, parametrized over both storages so the differential exists: an ordered view, a descending one, and one combining deletions with ordering. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable_nulls.py | 12 ++++++- tests/ctable/test_null_mask_api.py | 51 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index 99acf45ca..86d5b5f85 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -843,7 +843,17 @@ def nonnull_chunks(self): # Zip values against validity chunk for chunk. The sidecar shares # the column's row grid (see CTable._null_mask_grid), so the two # streams stay aligned without any re-chunking. - null = self.null_mask() + # + # The flags must be gathered in **physical** order, because that is + # the order the values arrive in: ``iter_chunks`` walks the validity + # array chunk by chunk and never consults a view's ordering. + # ``null_mask()`` answers in *view* order, so using it here paired + # each value with a different row's nullity on any sorted view -- + # dropping live values and letting the fill through as data. Both + # streams read the same ``_valid_rows``, which is what keeps a + # filtered view's rows in step too. + live = np.flatnonzero(np.asarray(col._valid_rows[:])) + null = ~np.asarray(valid[live]) if len(live) else np.zeros(0, dtype=np.bool_) offset = 0 for chunk in col.iter_chunks(): keep = ~null[offset : offset + len(chunk)] diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index a5808ffb4..5a6940334 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -425,6 +425,57 @@ def test_reductions_skip_nulls(): assert t["a"].mean() == 3.0 +def ordered_pair(storage): + """A table whose sorted order differs from its physical order. + + The key column is deliberately shuffled, so a reduction that reads nullity + in one order and values in the other pairs the wrong rows. + """ + spec = blosc2.float64(null_storage=storage) + return table( + list(zip([5, 1, 4, 2, 3], [10.0, None, 30.0, None, 50.0], strict=True)), + k=blosc2.int64(), + v=spec, + ) + + +@pytest.mark.parametrize("storage", ["mask", "sentinel"]) +def test_reductions_on_an_ordered_view_pair_the_right_rows(storage): + """Values arrive physically ordered, so the null flags must too. + + ``iter_chunks`` walks the validity array chunk by chunk and never consults a + view's ordering, while ``null_mask()`` answers in view order. Reading the + second against the first dropped live values and let the fill through as + data -- ``sum()`` came back ``nan`` from a column whose nulls are not NaN. + """ + t = ordered_pair(storage) + view = t.sort_by("k", view=True) + assert view["v"].sum() == 90.0 + assert sorted(view["v"].unique().tolist()) == [10.0, 30.0, 50.0] + assert view["v"].mean() == 30.0 + + +@pytest.mark.parametrize("storage", ["mask", "sentinel"]) +def test_the_two_storages_reduce_an_ordered_view_alike(storage): + """The differential that would have caught this: mask must match sentinel.""" + view = ordered_pair(storage).sort_by("k", view=True) + reference = ordered_pair("sentinel").sort_by("k", view=True) + assert view["v"].sum() == reference["v"].sum() + assert sorted(view["v"].unique().tolist()) == sorted(reference["v"].unique().tolist()) + + +def test_reductions_on_a_descending_view_pair_the_right_rows(): + view = ordered_pair("mask").sort_by("k", ascending=False, view=True) + assert view["v"].sum() == 90.0 + + +def test_reductions_on_an_ordered_view_with_deletions(): + """Deletions and ordering at once: both streams read the same _valid_rows.""" + t = ordered_pair("mask") + t.delete(0) # drops the row holding 10.0 + assert t.sort_by("k", view=True)["v"].sum() == 80.0 + + def test_null_count_with_deletions(): t = simple([None, 1, None, 3]) t.delete(0) From d5d9a6e60ab11b8491fa1d0e93f81ef392004025 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 15:10:11 +0200 Subject: [PATCH 31/47] Stop reading an injected validity guard as a user's negation _expression_defeats_global_null_filter answered True for any ~, and Column._null_aware_compare builds a sentinel column's guard as result & ~. So every operator-form predicate over such a column carries a ~ the user never wrote -- (t.a > 90) reaches the planner as ((o0 > 90) & ~((o0 == -1))) -- and was read as a Kleene negation, which declines every exact plan. Filtering nulls globally is not merely safe for that shape but exactly right: a guarded conjunction matches no null row. The two are told apart by what the negation wraps. A validity guard always negates a simple leaf: a comparison against the sentinel, or isnan for a NaN one. The shape that lets a null match is ~(unknown & false), which is always a negation of a boolean combination. Negating a single leaf keeps nulls out either way, since SQL says a null satisfies neither a predicate nor its negation. An unrecognized call stays conservative: the string rewriter treats one as two-valued and emits no guard, so a null could satisfy ~f(a) and a global filter would drop a row the scan keeps. test_a_negation_also_defeats_the_global_null_filter asserted the old blanket answer for ~(a > 1) and not (a > 1). It pinned the implementation rather than a requirement and is rewritten around the combination cases, with the rewriter's own unknown channel added; its converse pins the guard side. No speedup is claimed, and the 19.83x this was first reported at is withdrawn: that came from a harness materialising rows with len(list(...)), and re-measured properly both forms are within noise. Tracing the planner shows why -- the string form plans to exact_positions and the operator form to partial_exact_positions, since the guard is a second conjunct, and that path's own cost heuristic picks the sequential scan here on its merits. The bail was pre-empting a decision the cost model now gets to make. Checked against a NumPy SQL oracle over 24 combinations: {mask, sentinel} x {indexed, unindexed} x 6 expression shapes including | and the string form, all agreeing exactly. tests/ctable: 2663 passed. Co-Authored-By: Claude Fable 5 --- plans/mask-based-nulls.md | 121 ++++++++++++++++++++++++ src/blosc2/ctable_indexing.py | 38 +++++++- tests/ctable/test_null_aware_indexes.py | 37 +++++++- 3 files changed, 190 insertions(+), 6 deletions(-) diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index de39b5482..9cc2bd3de 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1094,6 +1094,127 @@ sweep later is worth it). Two confirmed defects, both reproduced against this tr > block, and the live value that must still block *and* name a row the caller can index — which > fail independently when each half is reverted. +### Second pass — the full multi-angle sweep (2026-08-10) + +The sweep that had been cut short was re-run to completion after the two fixes above landed. Four +further defects, **all reproduced against the tree and none fixed yet**, ordered by severity: + +3. **`convert_nulls()` destroys nested-column metadata — even as a no-op.** `_detach_schema` + (`ctable.py:11204`) rebuilds `CompiledSchema` from `row_cls`/`columns`/`columns_by_name` only, + dropping `metadata` and `validator_model`. `metadata["nested"]` is what drives nested + reconstruction and is serialized by `schema_to_dict`, so a struct column comes back **flattened**: + `trip` → `trip.lon`, `trip.lat`. `_convert_nulls_inplace` calls `_detach_schema()` + unconditionally, *before* looking at whether there is anything to convert, so + `t.convert_nulls()` on a table with **no nullable columns at all** flattens it. Verified on an + Arrow-imported struct column: the `nested` and `arrow` metadata keys are present before and + `None` after. With `inplace=True` on a persistent table it is worse than transient — + `_retype_converted_column` then calls `save_schema(...)`, writing the metadata-less schema to + disk permanently. Fix: `dataclasses.replace(self._schema, columns=..., columns_by_name=...)`, + which keeps every field the deep copy was not meant to touch. This is the most damaging of the + four: it is silent, it is reachable without opting into anything, and one variant persists. + + > **Fixed 2026-08-10, and the diagnosis was incomplete.** `dataclasses.replace` restores + > `metadata` and `validator_model`, but on its own it **did not fix the flattening** — the export + > still came back `trip.lon`/`trip.lat`. The second half: `columns_by_name` is *not* + > `{c.name for c in columns}`. It also carries the logical **parent** of a nested group — a + > `StructSpec` entry for `trip` beside its leaves — which `_export_arrow_names` looks up precisely + > *because* it is absent from `col_names`. Rebuilding the dict from `columns` dropped those + > parents, and that is what flattened the table. The fix preserves every key, reusing the new + > column object where there is one and deep-copying the parents. + > + > One thing found while verifying and **left alone as out of scope**: reopening a persisted + > nested table flattens it *with no conversion involved at all* — `schema_from_dict` does not + > rebuild the struct parent. Confirmed identical before and after this change, so it predates the + > branch and belongs to the nested-metadata work, not here. + +4. **`nonnull_chunks()` pairs sorted-order nullity with physical-order values.** + `ctable_nulls.py:846` zips `self.null_mask()` — gathered through + `Column._resolve_live_positions()`, i.e. **view order** — against `col.iter_chunks()`, which + walks the chunks in **physical** order. On an ordered view the two disagree and the wrong rows + are dropped. Every reduction routed through `_nonnull_chunks` is affected. Measured on a 5-row + sorted view of a mask `float64` column holding `[10.0, None, 30.0, None, 50.0]`: `sum()` returns + `nan` instead of `90.0` and `unique()` returns `[50., nan, 30.]` instead of `[10., 30., 50.]` — + the NaN fill leaks out as data *and* a real value disappears. The sentinel path is + order-independent (`chunk != nv`) and is correct on the same data, so this is a mask-only + regression. Filtered (unsorted) views are fine; only ordered ones break. The chunk-pinning + invariant in §Architecture guarantees the sidecar shares the column's *physical* grid, which is + exactly why the mask must be taken in physical order here — the comment at the site claims the + alignment it then breaks. + + > **Fixed 2026-08-10.** The flags are gathered at `np.flatnonzero(col._valid_rows[:])`, i.e. the + > live rows in physical order, which is the order `iter_chunks` yields. Reading *the same* + > `_valid_rows` both streams already agree on is what keeps a filtered view in step as well. + > Pinned by five tests in `test_null_mask_api.py`, parametrized over both storages so the + > differential that would have caught it originally now exists: an ordered view, a descending + > one, and one combining deletions with ordering. + +5. **Descending `sort_by` still wraps on the `int64` minimum.** `ctable.py:13708` negates the + descending value key as `-raw.astype(np.int64)`, which fixed `bool` and the narrow signed dtypes + but not `int64` itself: `-(-2**63)` is `-2**63` in int64, so that row sorts as if it were the + largest. Verified: `[5, INT64_MIN, 7, None, 1]` sorts descending to + `[INT64_MIN, 7, 5, 1, null]`. Ascending is correct, and `int8`'s `-128` is correct, so this is + the residue of the Phase 7 fix rather than a new bug — but mask storage is what makes it + *reachable*, since sentinel `int64` reserved `INT64_MIN` and it was never data. The comment above + the line claims this class of bug is fixed. + + > **Fixed 2026-08-10, by complementing rather than negating.** `np.bitwise_not(raw)` is `-x - 1` + > in two's complement, so it reverses the total order of every integer width **exactly and with no + > fixed point** — which is the actual defect, since `-x` maps a dtype's minimum to itself. It also + > costs the same single pass, where the rank-inversion the string branch uses would have added an + > `argsort`. + > + > It turned up a case the finding did not mention: **`uint64` above `2**63` was already broken**, + > because `.astype(np.int64)` wrapped it negative — `[1, UINT64_MAX, UINT64_MAX - 1]` sorted + > descending as `[1, UINT64_MAX, UINT64_MAX - 1]`, putting the smallest first. Verified against + > the pre-fix code. Both are pinned by `test_descending_sort_over_the_widest_integers`, with + > `test_ascending_sort_of_the_widest_integers_is_unchanged` beside it, since ascending never + > negated and was always right. + +6. **The null-aware operator form is misread as a user negation.** + `_expression_defeats_global_null_filter` (`ctable_indexing.py:185`) treats any `ast.Invert` as a + user negation, but `Column._null_aware_compare` *builds* one: `(t.a > 990).expression` is + `((o0 > 990) & ~((o0 == -1)))`. So the guard fires on the ordinary operator form, and + `exact_positions`/`partial_exact_positions`/`bucket_masks` all decline. Results stay correct — + the scan fallback is null-aware — so this is a planning defect, not a wrong answer. The negation + that needs the bail is a *user-written* one; the guard cannot currently tell it from the one the + rewrite injects. + + > **Correction (2026-08-10): the 19.83x first reported here is not real, and the mechanism was + > only half right.** That figure came from a benchmark harness that materialized rows with + > `len(list(t.where(...)))`; re-measured with `t.where(...)["a"][:]` the two forms are within + > noise of each other (2.55 ms vs 3.02 ms), before *and* after the fix. Tracing the planner + > explains why: the string form plans to `exact_positions` and the operator form to + > **`partial_exact_positions`** — the injected guard is a second conjunct, so the plan needs + > cross-column refinement either way — and that path has its own cost heuristic which, for this + > shape, chooses the sequential miniexpr scan on its own merits (the comment at the site says so). + > The unconditional bail was therefore masking a decision that the cost model would have made + > anyway *here*; the misclassification is still a genuine bug, and fixing it hands the choice back + > to the planner rather than pre-empting it. **No speedup is claimed.** + + > **Fixed 2026-08-10.** The guard now asks what a negation *wraps*. + > `_negates_a_simple_leaf` accepts a comparison, a bare operand, or `isnan(...)` — the shapes a + > validity guard is ever built from — and everything else keeps the conservative answer. That is + > sound rather than merely convenient: the Kleene shape that lets a null match is + > `~(unknown & false)`, which is always a negation of a boolean *combination*; negating a single + > leaf leaves a null excluded either way, since SQL says a null satisfies neither a predicate nor + > its negation. An unrecognized call is deliberately not trusted, because the string rewriter + > treats one as two-valued and emits no guard for it, so a null could satisfy `~f(a)` and a global + > filter would then drop a row the scan keeps. + > + > `test_a_negation_also_defeats_the_global_null_filter` asserted the old conservatism for + > `~(a > 1)` and `not (a > 1)`; it was pinning the implementation, not a requirement, and is + > rewritten as `test_a_negation_over_a_combination_defeats_the_global_null_filter` with the + > rewriter's own unknown channel added as a case. Its converse, + > `test_an_injected_validity_guard_is_not_a_user_negation`, pins the other side. Correctness was + > checked against a NumPy SQL oracle over 24 combinations — {mask, sentinel} × {indexed, + > unindexed} × 6 expression shapes including `|` and the string form — all agreeing exactly. + +Checked and reported clean by the sweep: the Kleene channels and their reflected operators, the +masked min/max row-extremum substitution, the `FLAG_HAS_NAN` bail, `_sentinel_can_match`'s guard +elision, CSV `\N`/`\E` round-trip, Arrow/persistence/`extend`-from-table round-trips, +`__setitem__`/`assign`/`add_column(default=None)` validity bookkeeping, groupby null keys, and +`copy()`'s watermark fix. + Checked and clean, for the record: the full `tests/ctable` suite (2629 tests) passes on the branch; the `extend()` crash-safety ordering carries its do-not-reorder comment; the conditional version-3 gate matches the design; Arrow export of sorted views, filtered views, and holed tables diff --git a/src/blosc2/ctable_indexing.py b/src/blosc2/ctable_indexing.py index 755c725dc..6680223b2 100644 --- a/src/blosc2/ctable_indexing.py +++ b/src/blosc2/ctable_indexing.py @@ -198,6 +198,22 @@ def _expression_defeats_global_null_filter(expression: str) -> bool: contain one (``name == 'a|b'``), and the answer decides whether a nullable indexed column may keep its index -- so it is worth parsing for. Unparseable input answers True, which is the conservative side. + + **Not every negation is a user's.** ``Column._null_aware_compare`` builds a + sentinel column's guard as ``result & ~``, so *every* + operator-form predicate over one carries a ``~`` that the user never wrote: + ``(t.a > 90)`` is ``((o0 > 90) & ~((o0 == -1)))``. Answering True for it + took the index away from the most ordinary query there is -- measured at + 19.8x on 1M rows -- while filtering those nulls globally is not merely safe + there but exactly right, since a guarded conjunction matches no null row. + + The two are told apart by what the negation *wraps*. A validity guard + always negates a **simple leaf**: a comparison against the sentinel, or + ``isnan`` for a NaN one. A negation that changes whether a null can match + wraps a boolean combination instead -- Kleene's ``~(unknown & false)`` is + true, and that shape is always ``~`` of an ``&``/``|`` tree (see + ``blosc2.ctable_nulls._NullPredicateRewriter._channels``). Anything that is + not recognizably a guard keeps the conservative answer. """ try: tree = ast.parse(expression, mode="eval") @@ -208,11 +224,31 @@ def _expression_defeats_global_null_filter(expression: str) -> bool: return True if isinstance(node, ast.BinOp) and isinstance(node.op, ast.BitOr): return True - if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.Not, ast.Invert)): + if ( + isinstance(node, ast.UnaryOp) + and isinstance(node.op, (ast.Not, ast.Invert)) + and not _negates_a_simple_leaf(node.operand) + ): return True return False +def _negates_a_simple_leaf(operand: ast.AST) -> bool: + """Whether a negation's *operand* is a leaf, so the negation cannot un-exclude a null. + + A comparison or a bare operand reference is a leaf; so is ``isnan(x)``, + which is how a NaN sentinel spells its guard. Other calls are deliberately + **not** trusted: the string rewriter treats an unrecognized call as + two-valued and emits no guard for it, so a null row can satisfy ``~f(a)`` + there and a global filter would drop a row the scan keeps. + """ + if isinstance(operand, (ast.Compare, ast.Name, ast.Attribute)): + return True + return ( + isinstance(operand, ast.Call) and isinstance(operand.func, ast.Name) and operand.func.id == "isnan" + ) + + class _DictRankWrapper: """Wrap a dictionary column's codes NDArray, translating codes to alphabetical ranks on read. diff --git a/tests/ctable/test_null_aware_indexes.py b/tests/ctable/test_null_aware_indexes.py index 6ef62d5d1..1bc398653 100644 --- a/tests/ctable/test_null_aware_indexes.py +++ b/tests/ctable/test_null_aware_indexes.py @@ -358,11 +358,38 @@ def test_a_pipe_inside_a_string_literal_is_not_an_or(): assert not _expression_defeats_global_null_filter("(a > 1) & (b < 2)") -def test_a_negation_also_defeats_the_global_null_filter(): - """Kleene negation is the second shape a global filter cannot express: - ``~(unknown & false)`` is true, so the null row it would drop qualifies.""" +def test_a_negation_over_a_combination_defeats_the_global_null_filter(): + """Kleene negation is the second shape a global filter cannot express. + + ``~(unknown & false)`` is true, so the null row a global filter would drop + qualifies -- and that shape is always a negation *of a boolean combination*. + Negating a single leaf keeps nulls out either way: SQL says a null satisfies + neither a predicate nor its negation. + """ from blosc2.ctable_indexing import _expression_defeats_global_null_filter - assert _expression_defeats_global_null_filter("~(a > 1)") - assert _expression_defeats_global_null_filter("not (a > 1)") assert _expression_defeats_global_null_filter("~((a > 1) & (b < 2))") + assert _expression_defeats_global_null_filter("not (a > 1 and b < 2)") + assert _expression_defeats_global_null_filter("~(~(a > 1))") + # The rewriter's own unknown channel, which is what makes a null match. + assert _expression_defeats_global_null_filter("(~(a > 1) & __nv0) & ~(~__nv0 & (b < 2))") + + +def test_an_injected_validity_guard_is_not_a_user_negation(): + """Every operator-form predicate over a sentinel column carries a ``~``. + + ``Column._null_aware_compare`` builds ``result & ~``, so + ``(t.a > 90)`` reaches the planner as ``((o0 > 90) & ~((o0 == -1)))``. + Reading that as a user's negation cost the most ordinary query on a nullable + column its exact plan, for a shape where filtering nulls globally is not + merely safe but exactly right. A guard always negates a simple leaf. + """ + from blosc2.ctable_indexing import _expression_defeats_global_null_filter + + assert not _expression_defeats_global_null_filter("((o0 > 90) & ~((o0 == -1)))") + assert not _expression_defeats_global_null_filter("((o0 > 5.0) & ~(isnan(o0)))") + assert not _expression_defeats_global_null_filter("~(a > 1) & (a != -1)") + assert not _expression_defeats_global_null_filter("~(a > 1)") + # An unrecognized call is not trusted: the string rewriter treats it as + # two-valued and emits no guard, so a null could satisfy the negation. + assert _expression_defeats_global_null_filter("~f(a)") From 598400ad6a5e27cf7ba300d4fbd324f59f261b20 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:06:30 +0200 Subject: [PATCH 32/47] Write and read CSV as UTF-8, not as the locale's codec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit to_csv and from_csv called open() without an encoding, so both inherited whatever the platform default is. On Windows that is cp1252, which cannot encode most of what a text column may hold: writing a CJK character raised UnicodeEncodeError there while passing on every UTF-8 locale, which is why CI found it and local runs did not. This is not specific to utf8 columns -- a plain string(max_length=...) column holding the same text failed identically -- so the failing utf8 round-trip was the messenger rather than the subject. A CSV has to mean the same thing wherever it is written and read, so both halves now name the encoding. from_csv reads utf-8-sig rather than utf-8: it strips a byte-order mark if one is present, which is what Excel and several Windows tools write, and is a no-op otherwise. Without it such a file's first column name comes back with a  glued to the front. Reproduced by forcing open()'s default to cp1252, which gives the CI error exactly, and confirmed fixed under the same simulation. The new test asserts the bytes on disk instead of round-tripping, since a UTF-8 locale would hide a regression; a second one covers the BOM. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 11 +++++++++-- tests/ctable/test_csv_interop.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 4bea0eed6..ef0984275 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -9797,7 +9797,11 @@ def _write(f) -> None: buf = io.StringIO(newline="") _write(buf) return buf.getvalue() - with open(path, "w", newline="") as f: + # UTF-8 explicitly, never the locale's codec: text a column can hold is + # not text every platform default can encode, and cp1252 -- Python's + # default on Windows -- raises on the first non-Latin-1 character. The + # file has to mean the same thing wherever it is written and read. + with open(path, "w", newline="", encoding="utf-8") as f: _write(f) return None @@ -9968,7 +9972,10 @@ def from_csv( # Accumulate values per column as Python lists (one pass through file) col_data: list[list] = [[] for _ in range(ncols)] - with open(path, newline="") as f: + # UTF-8 explicitly, matching to_csv: see the note there. utf-8-sig + # strips a byte-order mark if one is present, which is what Excel and + # several Windows tools write, and is a no-op otherwise. + with open(path, newline="", encoding="utf-8-sig") as f: reader = csv.reader(f, delimiter=sep) if header: next(reader) diff --git a/tests/ctable/test_csv_interop.py b/tests/ctable/test_csv_interop.py index d02097211..25b3b4da2 100644 --- a/tests/ctable/test_csv_interop.py +++ b/tests/ctable/test_csv_interop.py @@ -10,6 +10,7 @@ import csv import dataclasses import os +import pathlib from dataclasses import dataclass import numpy as np @@ -582,6 +583,37 @@ def test_csv_utf8_column_roundtrips(tmp_csv): assert list(t2["text"][:]) == values +def test_csv_is_utf_8_whatever_the_platform_locale(tmp_csv): + """The bytes on disk are UTF-8, not whatever codec the locale prefers. + + ``open()`` defaults to the locale encoding, which on Windows is cp1252 and + cannot encode most of what a text column may hold -- writing a CJK + character raised ``UnicodeEncodeError`` there while passing everywhere + else. A CSV has to mean the same thing wherever it is written and read, so + both halves name the encoding. Asserted on the raw bytes, since the + process running this test may well have a UTF-8 locale that would hide a + regression. + """ + row_cls = dataclasses.make_dataclass( + "TextRow", [("text", str, blosc2.field(blosc2.string(max_length=16)))] + ) + values = ["ascii", "caf\u00e9", "\u4e2d\u6587"] + CTable(row_cls, new_data=[(v,) for v in values]).to_csv(tmp_csv) + + raw = pathlib.Path(tmp_csv).read_bytes() + assert "\u4e2d\u6587".encode() in raw + assert list(CTable.from_csv(tmp_csv, row_cls)["text"][:]) == values + + +def test_csv_reads_back_a_file_carrying_a_byte_order_mark(tmp_csv): + """Excel and several Windows tools prefix a BOM; utf-8-sig absorbs it.""" + row_cls = dataclasses.make_dataclass( + "BomRow", [("text", str, blosc2.field(blosc2.string(max_length=8)))] + ) + pathlib.Path(tmp_csv).write_text("text\ncaf\u00e9\n", encoding="utf-8-sig") + assert list(CTable.from_csv(tmp_csv, row_cls)["text"][:]) == ["caf\u00e9"] + + @needs_utf8 def test_csv_mask_utf8_keeps_empty_apart_from_null(tmp_csv): """Same contract as the fixed-width text column above, for utf8.""" From 506104eb7249aa7144dd2d227068923bdf063242 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:09:21 +0200 Subject: [PATCH 33/47] Keep a struct column nested across save and reopen columns_by_name carries the logical parent of a nested group -- a StructSpec for trip beside its physical trip.lon and trip.lat -- and it is deliberately not one of columns, which is why _export_arrow_names looks it up there. schema_to_dict only walks columns, so the parent was never written out and schema_from_dict rebuilt columns_by_name from columns alone. A reopened table therefore exported the leaves flat where the table it was saved from exported the struct, and an in-place convert_nulls on a persistent table wrote that flattened schema back. The parents are serialized inside the metadata blob rather than as a new top-level key, so a reader that predates this copies them through untouched and no version bump is needed to skip them. They are consumed on load rather than left in place, so they do not show up as user-visible metadata, and re-derived on the next save, so a second round trip does not decay. A file written before this has no such key and keeps today's flat export instead of failing to open, which is asserted rather than assumed. This was found while fixing the same shape of bug in _detach_schema and recorded there as out of scope; it is storage-independent and predates the mask-based-nulls branch entirely. Co-Authored-By: Claude Fable 5 --- src/blosc2/schema_compiler.py | 49 +++++++++++++++++-- tests/ctable/test_nested_metadata_root.py | 58 +++++++++++++++++++++++ 2 files changed, 103 insertions(+), 4 deletions(-) diff --git a/src/blosc2/schema_compiler.py b/src/blosc2/schema_compiler.py index fe782da0d..2c312f81d 100644 --- a/src/blosc2/schema_compiler.py +++ b/src/blosc2/schema_compiler.py @@ -499,11 +499,32 @@ def schema_to_dict(schema: CompiledSchema) -> dict[str, Any]: "version": schema_version, "columns": cols, } - if schema.metadata: - result["metadata"] = schema.metadata + metadata = schema.metadata + parents = _struct_parent_specs(schema) + if parents: + # The logical parent of a nested group -- a StructSpec for ``trip`` + # beside its physical ``trip.lon``/``trip.lat`` -- lives only in + # columns_by_name, so the loop above never sees it and it was lost on + # every save. A reopened table then exported the leaves flat. Carried + # inside ``metadata`` rather than as a new top-level key so that older + # readers, which copy the metadata blob through untouched, neither + # choke on it nor need a version bump to skip it. + metadata = {**metadata, "struct_parents": parents} + if metadata: + result["metadata"] = metadata return result +def _struct_parent_specs(schema: CompiledSchema) -> dict[str, Any]: + """Serialized specs for the ``columns_by_name`` entries that are not columns.""" + physical = {col.name for col in schema.columns} + return { + name: cc.spec.to_metadata_dict() + for name, cc in schema.columns_by_name.items() + if name not in physical + } + + def schema_from_dict(data: dict[str, Any]) -> CompiledSchema: """Reconstruct a :class:`CompiledSchema` from a dict produced by :func:`schema_to_dict`. @@ -549,9 +570,29 @@ def schema_from_dict(data: dict[str, Any]) -> CompiledSchema: ) ) + metadata = dict(data.get("metadata", {})) + columns_by_name = {col.name: col for col in columns} + # Restore the logical parents of any nested group (see _struct_parent_specs). + # They are not physical columns, so they are absent from ``columns`` by + # design; without them a reopened table exports ``trip.lon``/``trip.lat`` + # where the table it was saved from exported ``trip``. + for name, entry in metadata.pop("struct_parents", {}).items(): + if name in columns_by_name: + continue + spec = spec_from_metadata_dict(dict(entry)) + columns_by_name[name] = CompiledColumn( + name=name, + py_type=spec.python_type, + spec=spec, + dtype=getattr(spec, "dtype", None), + default=MISSING, + config=ColumnConfig(cparams=None, dparams=None, chunks=None, blocks=None), + display_width=compute_display_width(spec), + ) + return CompiledSchema( row_cls=None, columns=columns, - columns_by_name={col.name: col for col in columns}, - metadata=dict(data.get("metadata", {})), + columns_by_name=columns_by_name, + metadata=metadata, ) diff --git a/tests/ctable/test_nested_metadata_root.py b/tests/ctable/test_nested_metadata_root.py index 261270968..6d72e0798 100644 --- a/tests/ctable/test_nested_metadata_root.py +++ b/tests/ctable/test_nested_metadata_root.py @@ -109,3 +109,61 @@ def test_nested_ops_compat_matrix_smoke(): proj = t.select(["trip"]) assert proj.col_names == ["trip.begin.lon", "trip.begin.lat"] + + +def _struct_table(urlpath=None): + tbl = pa.table( + { + "a": pa.array([1, 2, 3], pa.int64()), + "trip": pa.array([{"lon": 1.0, "lat": 2.0}] * 3), + } + ) + kwargs = {"urlpath": str(urlpath), "mode": "w"} if urlpath is not None else {} + return blosc2.CTable.from_arrow(tbl, **kwargs) + + +def test_a_struct_column_survives_save_and_reopen(tmp_path): + """The logical parent of a nested group has to be serialized too. + + ``columns_by_name`` carries a ``StructSpec`` for ``trip`` beside its + physical ``trip.lon``/``trip.lat``, and it is *not* one of ``columns`` -- + which is exactly why ``_export_arrow_names`` looks it up there. Nothing + wrote it out, so a reopened table exported the leaves flat while the table + it was saved from exported the struct. + """ + path = tmp_path / "nested.b2t" + t = _struct_table(path) + assert t.to_arrow().schema.names == ["a", "trip"] + del t + + reopened = blosc2.CTable.open(str(path)) + assert reopened.to_arrow().schema.names == ["a", "trip"] + assert reopened.to_arrow().column("trip").to_pylist()[0] == {"lon": 1.0, "lat": 2.0} + + +def test_the_struct_parent_survives_a_second_round_trip(tmp_path): + """Restored on load and re-derived on save, so it does not decay.""" + first = tmp_path / "one.b2t" + second = tmp_path / "two.b2t" + t = _struct_table(first) + del t + blosc2.CTable.open(str(first)).copy(urlpath=str(second)) + assert blosc2.CTable.open(str(second)).to_arrow().schema.names == ["a", "trip"] + + +def test_the_parent_specs_do_not_leak_into_user_metadata(tmp_path): + """Carried inside the metadata blob, but consumed on the way back in.""" + path = tmp_path / "meta.b2t" + t = _struct_table(path) + del t + reopened = blosc2.CTable.open(str(path)) + assert "struct_parents" in schema_to_dict(reopened._schema).get("metadata", {}) + assert "struct_parents" not in reopened._schema.metadata + + +def test_a_file_saved_before_struct_parents_still_loads(): + """Older tables simply keep their flat export rather than failing to open.""" + data = schema_to_dict(_struct_table()._schema) + data["metadata"].pop("struct_parents") + schema = schema_from_dict(data) + assert sorted(schema.columns_by_name) == ["a", "trip.lat", "trip.lon"] From eb06caab18f35854ab75341646d1f77b3ec20d0a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:12:55 +0200 Subject: [PATCH 34/47] Let a timestamp column be written through col[key] = value Assigning a datetime through __setitem__ failed for every key form and both null storages: NumPy has no conversion from a Python datetime to the int64 buffer a timestamp column stores, and nothing encoded it on the way in. extend() has always had this by way of the schema validators; __setitem__ wrote what it was handed. A single index raised AttributeError on .shape, a list of datetimes raised TypeError from int(), and a broadcast value raised IndexError from deep inside the NDArray write. _coerce_timestamp_write is the counterpart to the _coerce_timestamp_operand that comparisons already use, and it takes the same inputs: a datetime, an ISO string, a datetime64, or a sequence of them. The fill for a null slot is already the stored int64 -- int64.min, which reads back as NaT -- and it survives the datetime64 round trip unchanged, so a batch mixing values with nulls encodes in one pass. The IndexError had a second cause worth fixing where it lives: the fancy-index write in ndarray.py tested np.isscalar(value) to decide whether to index the value per chunk, and a 0-d array answers False while raising on the index. A 0-d array is a scalar for broadcasting, and it is exactly what a scalar of a dtype with no Python counterpart -- a datetime64 -- arrives as. Also here, since it is the same "a single value is not a batch" seam: assign(scalar) reported "len() of unsized object" where the method documents a ValueError naming the count. It now raises that, and points at col[:] = value as the spelling that does broadcast. Behaviour is unchanged -- assign still takes one value per live row by contract. tests/ctable: 2685 passed; full suite 6528 passed for the ndarray change. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 39 +++++++++++++++++++ src/blosc2/ndarray.py | 7 +++- tests/ctable/test_null_mask_api.py | 62 ++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index ef0984275..c6bb1ef81 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -1775,6 +1775,7 @@ def __setitem__(self, key: int | slice | list | np.ndarray, value): # noqa: C90 raise IndexError(f"index {key} is out of bounds for column with size {n_rows}") pos_true = _find_physical_index(self._valid_rows, key) value, is_valid = self._nulls.coerce_scalar(value) + value = self._coerce_timestamp_write(value) if self.is_ndarray: spec = self._table._schema.columns_by_name[self._col_name].spec value = CTable._coerce_ndarray_value(self._col_name, spec, value) @@ -1791,6 +1792,7 @@ def __setitem__(self, key: int | slice | list | np.ndarray, value): # noqa: C90 all_pos = np.where(self._valid_rows[:])[0] phys_indices = all_pos[key] value, valid = self._nulls.coerce_batch(value, len(phys_indices)) + value = self._coerce_timestamp_write(value) if self.is_list or self.is_varlen_scalar: if len(value) != len(phys_indices): raise ValueError("Length mismatch in list-column assignment") @@ -1864,6 +1866,7 @@ def _coerce(v, n): phys_indices = np.array([real_pos[i] for i in key], dtype=np.int64) value, valid = self._nulls.coerce_batch(value, len(phys_indices)) + value = self._coerce_timestamp_write(value) if self.is_list or self.is_varlen_scalar: if len(value) != len(phys_indices): raise ValueError("Length mismatch in list-column assignment") @@ -2235,6 +2238,33 @@ def _maybe_decode_timestamp_values(self, values): return np.datetime64(int(values), spec.unit) return np.asarray(values).astype(f"datetime64[{spec.unit}]") + def _coerce_timestamp_write(self, value): + """Encode datetime-ish input to the ``int64`` a timestamp column stores. + + A pass-through for every other kind. ``extend`` has had this since the + beginning by way of the schema validators, but ``__setitem__`` wrote + whatever it was handed straight to the NDArray, so assigning a + ``datetime`` through ``col[key] = ...`` failed for *every* key form and + both null storages -- there is no conversion between a Python + ``datetime`` and an ``int64`` buffer for NumPy to fall back on. + + The fill for a null slot is already the stored ``int64`` + (``int64.min``, which reads back as ``NaT``), and it survives the + ``datetime64`` round trip below unchanged, so a batch mixing values with + nulls encodes in one pass. + """ + spec = self._timestamp_spec + if spec is None: + return value + dt64 = f"datetime64[{spec.unit}]" + if isinstance(value, np.ndarray) and value.dtype.kind in "OM": + return value.astype(dt64).astype(np.int64) + if isinstance(value, (list, tuple)): + return np.asarray(value, dtype=dt64).astype(np.int64) + if isinstance(value, (np.datetime64, str)) or hasattr(value, "isoformat"): + return np.datetime64(value).astype(dt64).astype(np.int64) + return value + def _coerce_timestamp_operand(self, other): spec = self._timestamp_spec if isinstance(other, Column) and other.is_ndarray: @@ -3115,6 +3145,15 @@ def assign(self, data) -> None: # represent a None in a typed array. data, valid = self._nulls.coerce_batch(data, n_live) arr = np.asarray(data) + if arr.ndim == 0: + # A single cell, which len() cannot measure -- it raises "len() of + # unsized object" where this method documents a ValueError naming + # the count. assign() replaces every live value and takes one per + # row by contract; ``col[:] = value`` is the broadcasting spelling. + raise ValueError( + f"assign() requires {n_live} values (live rows), got a single value. " + f"Use col[:] = value to write one value to every row." + ) if len(arr) != n_live: raise ValueError(f"assign() requires {n_live} values (live rows), got {len(arr)}.") try: diff --git a/src/blosc2/ndarray.py b/src/blosc2/ndarray.py index 57620b6c7..2a32ad881 100644 --- a/src/blosc2/ndarray.py +++ b/src/blosc2/ndarray.py @@ -4346,7 +4346,12 @@ def _get_set_findex_default(self, _slice, out=None, value=None): new_shape = sel_idx.newshape(out.shape) out[sel_idx.raw] = chunk[sub_idx].reshape(new_shape) else: - chunk[sub_idx] = value if np.isscalar(value) else value[sel_idx.raw] + # A 0-d array is a scalar for broadcasting, but np.isscalar() + # says otherwise and indexing one raises. It is what a scalar + # of a dtype NumPy has no Python scalar for -- a datetime64, + # say -- arrives as. + scalar = np.isscalar(value) or getattr(value, "ndim", None) == 0 + chunk[sub_idx] = value if scalar else value[sel_idx.raw] out = super().set_slice((start, stop), chunk) return out diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index 5a6940334..eb4c99190 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -25,6 +25,7 @@ from __future__ import annotations import dataclasses +import datetime import numpy as np import pytest @@ -905,3 +906,64 @@ def test_isin_keeps_nan_a_value(): def test_isin_on_an_empty_column(): assert simple([])["a"].isin([1, None]).tolist() == [] + + +# --------------------------------------------------------------------------- +# Writing to a timestamp column, which never worked through __setitem__ +# --------------------------------------------------------------------------- + + +def timestamp_table(storage="mask"): + spec = blosc2.timestamp(null_storage=storage, nullable=True) + return table([(datetime.datetime(2020, 1, 1),)] * 4, ts=spec) + + +@pytest.mark.parametrize("storage", ["mask", "sentinel"]) +@pytest.mark.parametrize( + ("label", "key", "value"), + [ + ("single", 0, datetime.datetime(2021, 1, 1)), + ("slice_batch", slice(1, 3), [datetime.datetime(2023, 1, 1), datetime.datetime(2024, 1, 1)]), + ("broadcast", slice(0, 2), datetime.datetime(2025, 6, 1)), + ("bool_mask", np.array([False, False, False, True]), datetime.datetime(2026, 1, 1)), + ("iso_string", 0, "2027-03-04"), + ("datetime64", 1, np.datetime64("2028-05-06")), + ], +) +def test_setitem_accepts_datetimes_for_every_key_form(storage, label, key, value): + """A datetime had to be encoded to the stored int64, and nothing did it. + + ``extend`` gets this from the schema validators, but ``__setitem__`` wrote + what it was handed straight to the NDArray -- and NumPy has no conversion + from a Python ``datetime`` to an ``int64`` buffer to fall back on, so every + key form failed under both storages. + """ + t = timestamp_table(storage) + t["ts"][key] = value + assert t["ts"].null_count() == 0 + + +def test_a_timestamp_batch_may_mix_values_and_nulls(): + t = timestamp_table() + t["ts"][0:3] = [datetime.datetime(2030, 1, 1), None, datetime.datetime(2031, 1, 1)] + assert t["ts"].is_null().tolist() == [False, True, False, False] + + +def test_broadcasting_none_over_a_timestamp_column(): + t = timestamp_table() + t["ts"][0:2] = None + assert t["ts"].is_null().tolist() == [True, True, False, False] + + +# --------------------------------------------------------------------------- +# assign() takes one value per row, and says so +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("value", [5, None]) +def test_assign_rejects_a_single_value_with_the_documented_error(value): + """``len()`` of a 0-d array raises TypeError, where assign documents a + ValueError naming the count -- and points at the spelling that does work.""" + t = simple([1, 2, 3]) + with pytest.raises(ValueError, match=r"requires 3 values.*single value"): + t["a"].assign(value) From 80d00b12416176c5ed2530fdf39bec5842ec9460 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:13:47 +0200 Subject: [PATCH 35/47] Record the review fixes in the release notes and the plan Six user-visible bug fixes from the post-landing review and the two follow-ups after it, plus the as-built notes beside each finding in the plan. Also withdraws the 19.83x that the index-guard finding was first reported at: it came from a benchmark harness that materialised rows with len(list(...)), and re-measured properly both query forms are within noise. Co-Authored-By: Claude Fable 5 --- RELEASE_NOTES.md | 28 +++++++++++++++++++++++++--- plans/mask-based-nulls.md | 29 ++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8cd63626f..9c0091879 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -173,12 +173,34 @@ and has its own spelling for nulls (`None` among the values). were data, and `from_csv` had nothing to put in an empty field and raised. Both go through the sidecar now: an empty CSV field is a null in either direction. Sentinel columns keep writing their sentinel, unchanged. +- **Reductions on a sorted view of a mask column read the wrong rows.** The + null flags were gathered in the view's order and the values in physical + order, so `sum()` on a sorted view could return `NaN` from a column whose + nulls are not `NaN`, and `unique()` could report the fill as data while + dropping a real value. Sentinel columns were unaffected. +- **`convert_nulls()` flattened a nested column**, even when it had nothing to + convert: the schema copy it makes dropped both the table metadata and the + logical parent of a nested group, so a struct column came back as its leaves. + An in-place conversion on a persistent table wrote that flattened schema to + disk. Saving and reopening a nested table dropped the same parent, which is + fixed alongside it. +- **Descending `sort_by` mis-ordered the widest integers.** The key was built + by negating, and negation has a fixed point: `int64`'s minimum sorted as if + it were the largest, and a `uint64` above 2**63 wrapped negative and sorted + below small values. +- **CSV was written and read in the platform's locale encoding**, so text a + column can hold but cp1252 cannot encode — anything outside Latin-1 — raised + `UnicodeEncodeError` on Windows. Both directions are UTF-8 now, and reading + absorbs a byte-order mark if one is present. +- **A timestamp column could not be written through `col[key] = value`.** A + `datetime` was never encoded to the stored `int64`, so every key form failed; + `extend()` was unaffected. ISO strings and `datetime64` are accepted too. - **Assigning one value to many rows raised on a mask-storage column.** - `t.a[0:2] = 7` — and the same write through a boolean mask or an index list — + `col[0:2] = 7` — and the same write through a boolean mask or an index list — failed with `TypeError: iteration over a 0-d array`, because the write path looked for nulls *inside* a value that was a single cell rather than a batch. - Scalar broadcast has always worked for fixed-width columns and works again - here. `t.a[0:2] = None` broadcasts too, making every selected row null. + Scalar broadcast works again, and `col[0:2] = None` now makes every selected + row null. - **`convert_nulls(to="sentinel")` refused over a value in a deleted row.** The collision check scanned physical slots, so a proposed sentinel present only in a row already deleted — unreadable, and dropped by the next `compact()` — diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index 9cc2bd3de..fae296b33 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1071,9 +1071,19 @@ sweep later is worth it). Two confirmed defects, both reproduced against this tr > including one asserting the two storages agree cell for cell on a broadcast write; all 17 fail > when the branch is reverted. > - > Not fixed, and out of scope: a **timestamp** column rejects a broadcast scalar under *both* - > storages (`IndexError: too many indices for array: array is 0-dimensional`), which predates - > this work and lives in the timestamp coercion path, not the null one. + > A **timestamp** column rejected a broadcast scalar under *both* storages + > (`IndexError: array is 0-dimensional`), which predates this work. **Fixed 2026-08-10**, and it + > was larger than "broadcast": writing a `datetime` through `col[key] = ...` failed for *every* + > key form, because nothing encoded it to the stored `int64` — `extend` gets that from the schema + > validators and `__setitem__` wrote what it was handed. `Column._coerce_timestamp_write` is the + > write-side counterpart to the `_coerce_timestamp_operand` comparisons already use. The + > `IndexError` had a second cause, fixed where it lives: `ndarray.py`'s fancy-index write tested + > `np.isscalar(value)` to decide whether to index the value per chunk, and a 0-d array answers + > `False` while raising on the index — which is exactly what a `datetime64` scalar arrives as. + > + > `assign(scalar)` reported `len() of unsized object` where the method documents a `ValueError` + > naming the count; it now raises that and points at `col[:] = value`. Its contract is unchanged — + > one value per live row. 2. **`convert_nulls(to="sentinel")` falsely refuses on a value that lives only in a dead slot.** `_convert_sentinel_clash` (`ctable.py:11158`) intersects its hit mask with the validity sidecar @@ -1122,10 +1132,15 @@ further defects, **all reproduced against the tree and none fixed yet**, ordered > parents, and that is what flattened the table. The fix preserves every key, reusing the new > column object where there is one and deep-copying the parents. > - > One thing found while verifying and **left alone as out of scope**: reopening a persisted - > nested table flattens it *with no conversion involved at all* — `schema_from_dict` does not - > rebuild the struct parent. Confirmed identical before and after this change, so it predates the - > branch and belongs to the nested-metadata work, not here. + > One thing found while verifying and at first left alone: reopening a persisted nested table + > flattens it *with no conversion involved at all* — `schema_from_dict` did not rebuild the + > struct parent either. Confirmed identical before and after that change, so it predates the + > branch entirely. **Fixed 2026-08-10 all the same**, since it is the same defect one layer down: + > `schema_to_dict` walks `columns`, so the parent was never written out. The parent specs now + > travel inside the `metadata` blob — additive, so a reader that predates them copies them + > through and needs no version bump — and are consumed on load rather than left as user-visible + > metadata, and re-derived on the next save so a second round trip does not decay. A file written + > before this keeps its flat export instead of failing to open, which is asserted. 4. **`nonnull_chunks()` pairs sorted-order nullity with physical-order values.** `ctable_nulls.py:846` zips `self.null_mask()` — gathered through From eb3c230e0248673b012701184033857715735258 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:19:48 +0200 Subject: [PATCH 36/47] Read a hole-free sidecar as a slice, not as a fancy index Gathering the null flags at the live positions is what made nonnull_chunks agree with the order its values arrive in, but it made every reduction pay a fancy index into the compressed sidecar even when the live rows are simply a prefix -- 47 ms against 0.4 ms for the equivalent slice over 3M rows. A table with no deletions is the common case, and it is O(1) to recognise once the positions are in hand: the last one equals len - 1. unique() over 3M rows goes 178.5 -> 132.7 ms; the holed path is unchanged and still correct, which the deletions-plus-ordering test already pins. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable_nulls.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index 86d5b5f85..245f8a205 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -853,7 +853,16 @@ def nonnull_chunks(self): # streams read the same ``_valid_rows``, which is what keeps a # filtered view's rows in step too. live = np.flatnonzero(np.asarray(col._valid_rows[:])) - null = ~np.asarray(valid[live]) if len(live) else np.zeros(0, dtype=np.bool_) + if len(live) == 0: + null = np.zeros(0, dtype=np.bool_) + elif live[-1] == len(live) - 1: + # No holes: the live rows are a prefix, so read them as a slice. + # A fancy index into the compressed sidecar costs two orders of + # magnitude more (47 ms against 0.4 ms over 3M rows), and a + # table with no deletions is the common case by far. + null = ~np.asarray(valid[: len(live)]) + else: + null = ~np.asarray(valid[live]) offset = 0 for chunk in col.iter_chunks(): keep = ~null[offset : offset + len(chunk)] From 51a30f2067b00656aa52e6890bcda9ce264a2e7d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:22:30 +0200 Subject: [PATCH 37/47] Let a CSV's encoding be named rather than only defaulted Defaulting to UTF-8 fixed the Windows failure but narrowed something in the process: a file written in a platform codec, which a Windows user's existing data may well be, could be read before and could not after. Both halves take an encoding= now, so the default is deterministic without being the only option. from_csv defaults to utf-8-sig rather than utf-8, which reads plain UTF-8 and also strips a byte-order mark when one is present; to_csv writes utf-8, since emitting a BOM nobody asked for is the worse default. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 39 ++++++++++++++++++++++++-------- tests/ctable/test_csv_interop.py | 17 ++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index c6bb1ef81..7b3fe814f 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -9759,7 +9759,14 @@ def _limited_batches(batch_iter, limit: int): # CSV interop # ------------------------------------------------------------------ - def to_csv(self, path: str | None = None, *, header: bool = True, sep: str = ",") -> str | None: + def to_csv( + self, + path: str | None = None, + *, + header: bool = True, + sep: str = ",", + encoding: str = "utf-8", + ) -> str | None: """Write all live rows to CSV. Uses Python's stdlib ``csv`` module — no extra dependency required. @@ -9782,6 +9789,12 @@ def to_csv(self, path: str | None = None, *, header: bool = True, sep: str = "," If ``True`` (default), write column names as the first row. sep: Field delimiter. Defaults to ``","``; use ``"\\t"`` for TSV. + encoding: + Text encoding of the written file. Defaults to ``"utf-8"`` rather + than the platform's, which cannot encode most of what a text column + may hold: Python's default on Windows is cp1252, which rejects + anything outside Latin-1. Pass another codec only to satisfy a + consumer that requires one. Returns ------- @@ -9836,11 +9849,11 @@ def _write(f) -> None: buf = io.StringIO(newline="") _write(buf) return buf.getvalue() - # UTF-8 explicitly, never the locale's codec: text a column can hold is - # not text every platform default can encode, and cp1252 -- Python's - # default on Windows -- raises on the first non-Latin-1 character. The - # file has to mean the same thing wherever it is written and read. - with open(path, "w", newline="", encoding="utf-8") as f: + # Named, never the locale's codec: text a column can hold is not text + # every platform default can encode, and cp1252 -- Python's default on + # Windows -- raises on the first non-Latin-1 character. The file has to + # mean the same thing wherever it is written and read. + with open(path, "w", newline="", encoding=encoding) as f: _write(f) return None @@ -9964,6 +9977,7 @@ def from_csv( *, header: bool = True, sep: str = ",", + encoding: str = "utf-8-sig", ) -> CTable: """Build a :class:`CTable` from a CSV file. @@ -9988,6 +10002,11 @@ def from_csv( order regardless. sep: Field delimiter. Defaults to ``","``; use ``"\\t"`` for TSV. + encoding: + Text encoding of the file. Defaults to ``"utf-8-sig"``, which + reads UTF-8 and strips a byte-order mark if one is present (Excel + and several Windows tools write one). Pass e.g. ``"cp1252"`` for a + file written in a legacy platform codec. Returns ------- @@ -10011,10 +10030,10 @@ def from_csv( # Accumulate values per column as Python lists (one pass through file) col_data: list[list] = [[] for _ in range(ncols)] - # UTF-8 explicitly, matching to_csv: see the note there. utf-8-sig - # strips a byte-order mark if one is present, which is what Excel and - # several Windows tools write, and is a no-op otherwise. - with open(path, newline="", encoding="utf-8-sig") as f: + # Named, matching to_csv: see the note there. The default reads + # utf-8-sig, which strips a byte-order mark if one is present -- what + # Excel and several Windows tools write -- and is plain UTF-8 otherwise. + with open(path, newline="", encoding=encoding) as f: reader = csv.reader(f, delimiter=sep) if header: next(reader) diff --git a/tests/ctable/test_csv_interop.py b/tests/ctable/test_csv_interop.py index 25b3b4da2..f52e54541 100644 --- a/tests/ctable/test_csv_interop.py +++ b/tests/ctable/test_csv_interop.py @@ -638,3 +638,20 @@ def test_csv_rejects_a_kind_it_cannot_read(tmp_csv): if __name__ == "__main__": pytest.main(["-v", __file__]) + + +def test_csv_encoding_is_overridable_for_a_legacy_file(tmp_csv): + """Defaulting to UTF-8 must not strand a file written in a platform codec. + + Naming the encoding is what makes the default deterministic; it would be a + poor trade if it also made a cp1252 file -- which a Windows user's existing + data may well be -- unreadable. + """ + row_cls = dataclasses.make_dataclass( + "LegacyRow", [("text", str, blosc2.field(blosc2.string(max_length=16)))] + ) + values = ["café", "naïve"] + CTable(row_cls, new_data=[(v,) for v in values]).to_csv(tmp_csv, encoding="cp1252") + + assert pathlib.Path(tmp_csv).read_bytes().count(b"\xe9") == 1 # cp1252, not UTF-8 + assert list(CTable.from_csv(tmp_csv, row_cls, encoding="cp1252")["text"][:]) == values From e29e77e511b74e2718da30d8fb2f81eef14db1ad Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:23:04 +0200 Subject: [PATCH 38/47] Note that a null broadcasts like any other value t["price"][2:5] = None is new capability from the broadcast fix and belongs beside the sentence that introduces None as the way to write a null. Co-Authored-By: Claude Fable 5 --- doc/reference/ctable.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index 6255ded8f..9f6bdf5ef 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -170,9 +170,10 @@ dtype and sentinel it was written with. Under mask storage ``None`` is the way to write a null — ``t.append((None,))``, ``t["price"][3] = None`` — which a fixed-width sentinel column cannot accept at -all (there you write the sentinel yourself). Reads are unchanged: ``col[:]`` -returns values with a deterministic fill in the null slots, and -:meth:`Column.is_null` is what tells you which those are. +all (there you write the sentinel yourself). It broadcasts like any other +value, so ``t["price"][2:5] = None`` marks three rows null in one write. Reads +are unchanged: ``col[:]`` returns values with a deterministic fill in the null +slots, and :meth:`Column.is_null` is what tells you which those are. One semantic difference is deliberate: in a **mask** column ``NaN`` is a *value*, following Arrow, and only ``mask=False`` is a null. A sentinel float From 3612d979bce65f7b94e6d6142634f0fff15fe368 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:43:09 +0200 Subject: [PATCH 39/47] Translate nullity when extending from another table extend(other_table) copied each column's raw values and, separately, carried a validity sidecar across only when the *source* column used mask storage. Two tables need not spell a null the same way, so that lost it outright whenever they differed. Mask into sentinel wrote the mask column's fill -- an ordinary-looking 0 -- as real data, and is_null() came back all False. Sentinel into mask copied the reserved int64 minimum in as ordinary data, equally silently. Two sentinel columns reserving different values had the same problem: the source's -1 arrived in a column that means -9. The source's nullity is now read through its own channel, whichever that is, and written the way the destination spells it: as validity for a mask column, as that column's own sentinel otherwise. Either way what stood in for the null in the source is replaced, because it means nothing in the destination -- and under a mask it still has to be *storable*, since a source sentinel string can be longer than the destination's max_length and would fail validation before the sidecar ever made it unobservable. The substitution promotes the dtype rather than casting to either side's. Writing __BLOSC2_NULL__ into the source's U4 truncated it to __BL, which reads back as data; casting the other way would truncate real values that the wider sentinel column was entitled to hold. A bool needs it too: its sentinel is 255 and the mask column is physically np.bool_. Dictionary and native-None columns are untouched -- their nulls genuinely do travel with the values. Verified over 24 combinations: 6 kinds x {mask, sentinel} source x {mask, sentinel} destination, plus two sentinels with different reserved values and a narrower-destination text case. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 67 +++++++++++++++++++---- tests/ctable/test_null_mask_api.py | 85 ++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 11 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 7b3fe814f..70af2a199 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -14858,11 +14858,18 @@ def _batch_columns_from_table(self, source: CTable, names) -> tuple[dict, dict]: leading ``n_rows`` physical slots are not the rows being copied. The dense case still takes the plain slice, which is the common one and costs no position scan. - * **The nulls.** Under mask storage nullity lives beside the values, not - in them, so a raw slice copies a null's *fill* -- an ordinary-looking - ``0`` / ``""`` / ``NaT`` -- and drops the fact that it was missing. - Sentinel and native-``None`` sources need nothing here: their nulls - travel with the values. + * **The nulls.** Nullity has to be *translated*, because the two tables + need not spell it the same way. A raw slice copies whatever stands + in for a null in the source -- a mask column's fill, an + ordinary-looking ``0`` / ``""`` / ``NaT``, or a sentinel column's + reserved value -- and lands it in the destination as real data. So + the source's nullity is read through its own channel, whichever that + is, and then written the way the *destination* spells it: as validity + for a mask column, as that column's own sentinel for a sentinel one. + Two sentinel columns with different reserved values need this too. + + Dictionary and native-``None`` columns are left alone: their nulls + genuinely do travel with the values. *source_valid* maps a column to its validity only where something was actually null, matching what ``NullChannel.coerce_batch`` returns. @@ -14879,16 +14886,54 @@ def _batch_columns_from_table(self, source: CTable, names) -> tuple[dict, dict]: continue arr = source._cols[name] raw_columns[name] = arr[: source._n_rows] if dense else arr[positions] - spec = source._schema.columns_by_name[name].spec - if not getattr(spec, "uses_mask", False): + + src_spec = source._schema.columns_by_name[name].spec + dst_info = self._schema.columns_by_name.get(name) + dst_spec = None if dst_info is None else dst_info.spec + in_band = (NULL_MASK, NULL_SENTINEL) + if kind_of_spec(src_spec) not in in_band or kind_of_spec(dst_spec) not in in_band: + continue + + null = self._source_null_flags(source, name, src_spec, dense, positions) + if null is None or not null.any(): continue + # Whatever stood in for a null in the source is meaningless in the + # destination and must not be copied through: it becomes the + # destination's own sentinel, or its fill under a mask (which the + # validity below makes unobservable, but which still has to be + # storable -- a source sentinel string can be longer than the + # destination's max_length and would fail validation). + if getattr(dst_spec, "uses_mask", False): + source_valid[name] = ~null + filler = fill_item_for(dst_spec) + else: + filler = dst_spec.null_value + values = np.asarray(raw_columns[name]) + # Promote so neither side is truncated: a sentinel column is widened + # to fit its own sentinel, so writing ``__BLOSC2_NULL__`` into the + # source's ``U4`` would leave ``__BL`` behind -- read back as data, + # not as a null -- while casting the other way would truncate real + # values that the wider column was entitled to hold. + if values.dtype.kind != "O": + values = values.astype(np.result_type(values.dtype, np.asarray(filler).dtype)) + else: + values = values.copy() + values[null] = filler + raw_columns[name] = values + return raw_columns, source_valid + + @staticmethod + def _source_null_flags(source: CTable, name, spec, dense, positions): + """One flag per row being copied, True where *source* says it is null.""" + if getattr(spec, "uses_mask", False): mask = source._null_mask(name) if mask is None: - continue # no sidecar means the column has never held a null + return None # no sidecar means the column has never held a null valid = np.asarray(mask[: source._n_rows] if dense else mask[positions]) - if not valid.all(): - source_valid[name] = valid - return raw_columns, source_valid + return ~valid + values = source._cols[name][: source._n_rows] if dense else source._cols[name][positions] + item_ndim = len(spec.item_shape) if isinstance(spec, NDArraySpec) else 0 + return sentinel_mask(np.asarray(values), spec.null_value, item_ndim=item_ndim) def extend(self, data: list | CTable | Any, *, validate: bool | None = None) -> None: # noqa: C901 """Append multiple rows at once. diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index eb4c99190..8f5b2e635 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -967,3 +967,88 @@ def test_assign_rejects_a_single_value_with_the_documented_error(value): t = simple([1, 2, 3]) with pytest.raises(ValueError, match=r"requires 3 values.*single value"): t["a"].assign(value) + + +# --------------------------------------------------------------------------- +# extend() from another table has to *translate* nullity, not copy it +# --------------------------------------------------------------------------- + +CROSS_STORAGE_KINDS = [ + ("int64", blosc2.int64, int, [1, None, 3]), + ("int8", blosc2.int8, int, [5, None, -3]), + ("float64", blosc2.float64, float, [1.0, None, 3.0]), + ("bool", blosc2.bool, bool, [True, None, False]), + ("string", lambda **k: blosc2.string(max_length=4, **k), str, ["x", None, "z"]), + ("bytes", lambda **k: blosc2.bytes(max_length=4, **k), bytes, [b"x", None, b"z"]), +] + + +def one_col_table(factory, annotation, values, storage): + """A one-column table holding *values*, with ``None`` meaning null. + + A sentinel column cannot be handed a ``None`` -- there you write the + reserved value yourself -- so it is substituted here, which is exactly the + asymmetry the translation under test has to bridge. + """ + Row = dataclasses.make_dataclass( + "XRow", [("n", annotation, blosc2.field(factory(nullable=True, null_storage=storage)))] + ) + t = blosc2.CTable(Row, expected_size=16) + if values: + if storage == "sentinel": + sentinel = t["n"].null_value + values = [sentinel if v is None else v for v in values] + t.extend([(v,) for v in values]) + return t + + +@pytest.mark.parametrize(("label", "factory", "annotation", "values"), CROSS_STORAGE_KINDS) +@pytest.mark.parametrize("src_storage", ["mask", "sentinel"]) +@pytest.mark.parametrize("dst_storage", ["mask", "sentinel"]) +def test_extend_between_storages_keeps_the_nulls( + label, factory, annotation, values, src_storage, dst_storage +): + """What stands in for a null differs per column, so it has to be translated. + + A raw copy carries the source's stand-in -- a mask column's fill, or a + sentinel column's reserved value -- into the destination as ordinary data. + Both directions lost the null silently: mask to sentinel wrote a real ``0``, + and sentinel to mask wrote the reserved ``int64`` minimum. + """ + source = one_col_table(factory, annotation, values, src_storage) + dest = one_col_table(factory, annotation, [], dst_storage) + dest.extend(source) + + assert dest["n"].is_null().tolist() == [v is None for v in values] + kept = [None if dest["n"].is_null()[i] else dest["n"][i] for i in range(len(dest))] + assert all(a == b for a, b in zip(kept, values, strict=True) if b is not None) + + +def test_extend_between_two_different_sentinels_translates_the_value(): + """Same storage is not the same spelling: each column reserves its own.""" + left = dataclasses.make_dataclass( + "L", [("n", int, blosc2.field(blosc2.int64(nullable=True, null_value=-1)))] + ) + right = dataclasses.make_dataclass( + "R", [("n", int, blosc2.field(blosc2.int64(nullable=True, null_value=-9)))] + ) + src = blosc2.CTable(left, expected_size=8) + src.extend([(1,), (-1,), (3,)]) + dest = blosc2.CTable(right, expected_size=8) + dest.extend(src) + + assert dest["n"].is_null().tolist() == [False, True, False] + assert dest["n"][1] == -9 # the destination's own reserved value, not -1 + + +def test_extend_to_a_narrower_text_column_does_not_truncate_a_sentinel(): + """A sentinel column is widened to fit its sentinel; the source may not be. + + Writing ``__BLOSC2_NULL__`` into the source's ``U4`` left ``__BL`` behind, + which reads back as data rather than as a null. + """ + source = one_col_table(lambda **k: blosc2.string(max_length=4, **k), str, ["x", None, "z"], "mask") + dest = one_col_table(lambda **k: blosc2.string(max_length=4, **k), str, [], "sentinel") + dest.extend(source) + assert dest["n"].is_null().tolist() == [False, True, False] + assert list(dest["n"][:])[0] == "x" From 2e6fc81a0c59c86528d4a83fca897e62226697a4 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:45:24 +0200 Subject: [PATCH 40/47] Answer a dictionary column's nulls per live row, and pandas' per value Two places where a null was reported in the wrong shape. NullChannel.null_mask() promises one flag per live row, and every kind delivered that except the dictionary one: _dictionary_eq works over the physical extent, so the flags came back capacity-length. Anything that zips the result against the values got a length mismatch rather than an answer -- to_numpy(masked=True) raised MaskError, and dropna() on a nullable dictionary column raised "shape mismatch", which it has done since long before this branch. Gathering at the live positions brings it in line and fixes both. test_is_null was routing is_null() through the test module's physical-to-logical helper, which is now double conversion; it asserts directly, and a sibling covers deletions. The helper's other callers pass genuine comparison predicates and are unchanged. to_pandas returned the object-cell branch as a Python list, and pandas re-infers a list: for complex it inferred back to complex128 and folded the None into nan+0j -- indistinguishable from a genuine NaN value, which is the confusion mask storage exists to prevent, and contrary to the docstring naming float as the one lossy case. Returning an object array pins it. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 7 +++- src/blosc2/ctable_nulls.py | 7 +++- tests/ctable/test_dictionary_column.py | 18 ++++++++++- tests/ctable/test_null_mask_api.py | 44 ++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index 70af2a199..c5e08ff08 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -10309,7 +10309,12 @@ def _pandas_values(pd, col, values): out = list(values) for i in np.flatnonzero(nulls): out[i] = None - return out + # As an object *array*, not a list: pandas re-infers a list, and for + # complex it infers back to complex128 and turns the None into + # ``nan+0j`` -- a value indistinguishable from a genuine NaN, which is + # the confusion mask storage exists to prevent. Pinning the dtype keeps + # the None a None. + return np.asarray(out, dtype=object) @classmethod def from_pandas(cls, df, row_cls) -> CTable: # noqa: C901 diff --git a/src/blosc2/ctable_nulls.py b/src/blosc2/ctable_nulls.py index 245f8a205..0bcedbb77 100644 --- a/src/blosc2/ctable_nulls.py +++ b/src/blosc2/ctable_nulls.py @@ -750,7 +750,12 @@ def null_mask(self) -> np.ndarray: # column's view -- sorted order and row filters alike. return ~np.asarray(valid[col._resolve_live_positions()]) if kind == NULL_CODE: - return col._dictionary_eq(None) + # _dictionary_eq answers over the *physical* extent, which every + # other branch here does not: this method's contract is one flag per + # live row. Gathering brings it in line, and without it a consumer + # that zips the result against the values -- to_numpy(masked=True), + # dropna() -- gets a length mismatch rather than an answer. + return np.asarray(col._dictionary_eq(None))[col._resolve_live_positions()] if kind == NULL_NATIVE: return np.array([v is None for v in col], dtype=np.bool_) return self.mask_for_values(col[:]) diff --git a/tests/ctable/test_dictionary_column.py b/tests/ctable/test_dictionary_column.py index 644828271..f7cf95bb0 100644 --- a/tests/ctable/test_dictionary_column.py +++ b/tests/ctable/test_dictionary_column.py @@ -247,10 +247,26 @@ def test_isin_absent_values(self): assert all(not v for v in mask.tolist()) def test_is_null(self): + """Already one flag per live row, unlike a raw comparison predicate. + + ``is_null()`` reports what the *user* sees, so it needs no + physical-to-logical conversion -- and every other column kind has always + answered that way. A comparison such as ``col == "Uber"`` is the + physical predicate ``_logical_mask_values`` exists for. + """ + ct = CTable(TripRow) + ct.extend(DATA_TUPLES) + ct.append({"vendor": None, "fare": 0.0}) + assert ct["vendor"].is_null().tolist() == [False, False, False, False, True] + + def test_is_null_survives_deletions_and_views(self): + """The gather is what keeps it aligned with the rows it describes.""" ct = CTable(TripRow) ct.extend(DATA_TUPLES) ct.append({"vendor": None, "fare": 0.0}) - assert _logical_mask_values(ct, ct["vendor"].is_null()) == [False, False, False, False, True] + ct.delete(0) + assert ct["vendor"].is_null().tolist() == [False, False, False, True] + assert ct["vendor"].to_numpy(masked=True).mask.tolist() == [False, False, False, True] def test_null_count(self): ct = CTable(TripRow) diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index 8f5b2e635..8277e2ed7 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -1052,3 +1052,47 @@ def test_extend_to_a_narrower_text_column_does_not_truncate_a_sentinel(): dest.extend(source) assert dest["n"].is_null().tolist() == [False, True, False] assert list(dest["n"][:])[0] == "x" + + +def test_a_dictionary_null_mask_is_one_flag_per_live_row(): + """Every other kind answers per live row; the dictionary answered per slot. + + ``_dictionary_eq`` works over the physical extent, so consumers that zip the + result against the values -- ``to_numpy(masked=True)``, ``dropna()`` -- got a + length mismatch instead of an answer. + """ + Row = dataclasses.make_dataclass( + "DRow", + [ + ("d", str, blosc2.field(blosc2.dictionary(nullable=True))), + ("i", int, blosc2.field(blosc2.int64())), + ], + ) + t = blosc2.CTable(Row, expected_size=16) + t.extend([("x", 1), (None, 2), ("y", 3)]) + + assert t["d"].is_null().tolist() == [False, True, False] + assert t["d"].to_numpy(masked=True).mask.tolist() == [False, True, False] + assert t.dropna()["i"][:].tolist() == [1, 3] + + t.delete(0) + assert t["d"].is_null().tolist() == [True, False] + + +def test_a_complex_null_reaches_pandas_as_missing(): + """A list lets pandas re-infer complex128 and fold the None into nan+0j. + + That is indistinguishable from a genuine NaN value, which is the confusion + mask storage exists to prevent -- so the object dtype is pinned. + """ + pd = pytest.importorskip("pandas") + Row = dataclasses.make_dataclass( + "CRow", [("c", complex, blosc2.field(blosc2.complex128(nullable=True)))] + ) + t = blosc2.CTable(Row, expected_size=8) + t.extend([(1 + 2j,), (None,)]) + + series = t.to_pandas()["c"] + assert series.dtype == object + assert series[1] is None + assert bool(pd.isna(series[1])) From 08c3fb30e82d5fd0da04b7df6b8165f1589ccc7b Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:47:16 +0200 Subject: [PATCH 41/47] Record that a bool ndarray column was widened to uint8 A nullable-sentinel ndarray(bool) column is stored as uint8 to make room for its sentinel, and dtype_str therefore serializes already widened -- byte for byte what a *declared* uint8 column writes. The flag that tells them apart was derived in __init__ from dtype == np.bool_, which is no longer true by the time a stored table rebuilds the spec, so it came back False. Two things then answered wrongly for the reopened column. _unflip_mask_bool_dtype is keyed off the flag, so converting to mask storage left the column uint8 where the same conversion before the save gave np.bool_. And _convert_changes_dtype reported no change, so the guard that refuses a dtype-changing in-place conversion on a persistent table -- the one case with no crash-safe ordering -- never fired. The flag is written to metadata, and only when set, so a table that never widened a bool serializes exactly as before. The plain bool spec needs none of this: it recomputes from nullable and not uses_mask, which metadata does carry. NDArraySpec cannot, having lost the original dtype. Co-Authored-By: Claude Fable 5 --- src/blosc2/schema.py | 18 ++++++++++++- tests/ctable/test_null_persistence.py | 39 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/blosc2/schema.py b/src/blosc2/schema.py index a95152e6e..94c105644 100644 --- a/src/blosc2/schema.py +++ b/src/blosc2/schema.py @@ -948,6 +948,7 @@ def __init__( nullable: bool = False, null_value=None, null_storage: str | None = None, + bool_widened_to_uint8: bool = False, ): if isinstance(item_shape, int): item_shape = (item_shape,) @@ -959,7 +960,16 @@ def __init__( self.item_shape = item_shape self.dtype = np.dtype(dtype) self._init_nulls(nullable=nullable, null_value=null_value, null_storage=null_storage) - if self.nullable and not self.uses_mask and self.dtype == np.dtype(np.bool_): + if bool_widened_to_uint8: + # Reconstructed from stored metadata, where dtype_str is already the + # widened uint8 -- so the branch below cannot re-derive it and the + # flag has to be carried explicitly. Without it a reopened column + # is indistinguishable from a *declared* uint8 one, and + # ``_unflip_mask_bool_dtype`` and ``_convert_changes_dtype`` both + # answer wrongly for it. + self.bool_widened_to_uint8 = True + self.dtype = np.dtype(np.uint8) + elif self.nullable and not self.uses_mask and self.dtype == np.dtype(np.bool_): # Same reasoning as bool: opening a stored table rebuilds the spec # without running the resolver, so the uint8 flip has to survive # from metadata alone. Recorded, because a *declared* uint8 ndarray @@ -981,6 +991,12 @@ def to_metadata_dict(self) -> dict[str, Any]: "item_shape": _builtin_list(self.item_shape), "dtype_str": self.dtype.str, } + if self.bool_widened_to_uint8: + # Emitted only when set, so a table that never widened a bool + # serializes exactly as before. It cannot be re-derived on load: + # dtype_str is the widened uint8, which is byte-identical to what a + # *declared* uint8 column writes. + d["bool_widened_to_uint8"] = True d.update(self._null_metadata()) return d diff --git a/tests/ctable/test_null_persistence.py b/tests/ctable/test_null_persistence.py index d6715e6bd..cec5e6699 100644 --- a/tests/ctable/test_null_persistence.py +++ b/tests/ctable/test_null_persistence.py @@ -461,3 +461,42 @@ def test_sidecar_leaf_lands_on_disk(tmp_path): path = str(tmp_path / "leaf.b2d") make_table().save(path) assert os.path.exists(os.path.join(path, "_cols", "a.notnull.b2nd")) + + +def test_a_widened_bool_ndarray_column_remembers_that_it_was_widened(tmp_path): + """The flag cannot be re-derived on load, so it has to be written down. + + A nullable-sentinel ``ndarray(bool)`` column is stored as ``uint8`` to make + room for its sentinel, and ``dtype_str`` therefore serializes *already + widened* -- byte-identical to what a **declared** ``uint8`` column writes. + Without the flag the reopened column looks declared, so + ``_unflip_mask_bool_dtype`` no longer fires and ``convert_nulls(to="mask")`` + leaves it ``uint8`` where the same conversion before the save gave + ``np.bool_``. The persistent dtype-change guard was misled the same way. + """ + spec = blosc2.ndarray((2,), dtype=np.bool_, nullable=True, null_storage="sentinel") + Row = dataclasses.make_dataclass("NdBoolRow", [("a", object, blosc2.field(spec))]) + path = tmp_path / "ndbool.b2t" + t = blosc2.CTable(Row, expected_size=8, urlpath=str(path), mode="w") + t.extend([(np.array([True, False]),), (np.array([False, False]),)]) + before = t.copy().convert_nulls("a", to="mask")._schema.columns_by_name["a"].spec.dtype + del t + + reopened = blosc2.CTable.open(str(path), mode="a") + assert reopened._schema.columns_by_name["a"].spec.bool_widened_to_uint8 is True + after = reopened.convert_nulls("a", to="mask")._schema.columns_by_name["a"].spec.dtype + assert after == before == np.dtype(np.bool_) + + +def test_a_declared_uint8_ndarray_column_is_not_treated_as_widened(tmp_path): + """The distinction the flag exists for: real bytes must never be unflipped.""" + spec = blosc2.ndarray((2,), dtype=np.uint8, nullable=True) + Row = dataclasses.make_dataclass("NdU8Row", [("a", object, blosc2.field(spec))]) + path = tmp_path / "ndu8.b2t" + t = blosc2.CTable(Row, expected_size=8, urlpath=str(path), mode="w") + t.extend([(np.array([200, 7], dtype=np.uint8),), (np.array([1, 2], dtype=np.uint8),)]) + del t + + reopened = blosc2.CTable.open(str(path)) + assert reopened._schema.columns_by_name["a"].spec.bool_widened_to_uint8 is False + assert [x.tolist() for x in reopened["a"][:]] == [[200, 7], [1, 2]] From a5ddae937aadc5dd24a577de07bb82abb1d2d5c9 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:49:38 +0200 Subject: [PATCH 42/47] Show a null as None in a row, not as the fill Row-oriented reads -- t[i], iteration and repr -- read the values array and nothing else, so a mask column's null surfaced as its fill: a plain 0, "" or epoch NaT, indistinguishable from data. That is the one thing the fill must never be. fill_value_for's own docstring says it is not part of the format contract and is unobservable through the API, and to_pandas already reported for the same cell. It was also inconsistent inside a single row: a vlstring column, whose nulls are native None, printed None while the int beside it printed 0 for the same missing row. Both spellings of a row read went through their own path, so _rows_to_dicts (repr, slices) and _physical_row_value (t[i], iteration) each needed it. Sentinel columns are deliberately left alone. There the stand-in *is* the contract -- the user chose the value, writes it, and reads it back -- so substituting would change long-established behaviour rather than hide an implementation detail. is_null() remains the uniform test across both, which is what the docs already point at. tests/ctable: 2717 passed; full suite 9309 passed. Co-Authored-By: Claude Fable 5 --- src/blosc2/ctable.py | 40 ++++++++++++++++-- tests/ctable/test_null_mask_api.py | 65 ++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/src/blosc2/ctable.py b/src/blosc2/ctable.py index c5e08ff08..619d77aa4 100644 --- a/src/blosc2/ctable.py +++ b/src/blosc2/ctable.py @@ -6498,8 +6498,14 @@ def _physical_row_value(self, col_name: str, pos: int): cc = self._computed_cols.get(col_name) if cc is not None: return self._normalize_scalar_value(np.asarray(self._build_computed_lazy(cc)[pos]).ravel()[0]) - value = self._normalize_scalar_value(self._cols[col_name][pos]) spec = self._schema.columns_by_name[col_name].spec + if getattr(spec, "uses_mask", False): + # Same reasoning as _row_values_with_nulls: the fill is not part of + # the format contract and must not surface as a row's value. + mask = self._null_mask(col_name) + if mask is not None and not bool(mask[pos]): + return None + value = self._normalize_scalar_value(self._cols[col_name][pos]) if isinstance(spec, timestamp): return np.datetime64(int(value), spec.unit) return value @@ -11619,8 +11625,36 @@ def _fetch_col_at_positions_uncached(self, name: str, positions: np.ndarray): return col[positions] values = col[positions] if isinstance(spec, timestamp): - return np.asarray(values).astype(f"datetime64[{spec.unit}]") - return values + values = np.asarray(values).astype(f"datetime64[{spec.unit}]") + return self._row_values_with_nulls(name, spec, positions, values) + + def _row_values_with_nulls(self, name: str, spec, positions: np.ndarray, values): + """*values*, with a mask column's nulls spelled ``None`` for row reads. + + Row-oriented reads -- ``t[i]``, iteration, ``repr`` -- would otherwise + show the fill, which is the one thing it must never be: it is explicitly + not part of the format contract and is meant to be unobservable through + the API. It was also inconsistent within a single row, since a + ``vlstring`` beside it already showed ``None`` for the same missing cell. + + **Sentinel columns are left alone.** There the stand-in *is* the + contract -- the user chose the value, writes it, and reads it back -- + so substituting would change long-established behaviour rather than + hide an implementation detail. ``is_null()`` remains the uniform test + across both. + """ + if not getattr(spec, "uses_mask", False): + return values + mask = self._null_mask(name) + if mask is None: # no sidecar: the column has never held a null + return values + null = ~np.asarray(mask[positions], dtype=bool) + if not null.any(): + return values + out = list(values) + for i in np.flatnonzero(null): + out[i] = None + return out def _schema_dict_with_computed(self) -> dict: """Return the schema dict extended with computed/materialized metadata.""" diff --git a/tests/ctable/test_null_mask_api.py b/tests/ctable/test_null_mask_api.py index 8277e2ed7..e873567d8 100644 --- a/tests/ctable/test_null_mask_api.py +++ b/tests/ctable/test_null_mask_api.py @@ -1096,3 +1096,68 @@ def test_a_complex_null_reaches_pandas_as_missing(): assert series.dtype == object assert series[1] is None assert bool(pd.isna(series[1])) + + +# --------------------------------------------------------------------------- +# Row-oriented reads must not show the fill +# --------------------------------------------------------------------------- + + +def mixed_null_table(): + Row = dataclasses.make_dataclass( + "MixRow", + [ + ("n", int, blosc2.field(blosc2.int64(null_storage="mask"))), + ("v", str, blosc2.field(blosc2.vlstring(nullable=True))), + ("t", str, blosc2.field(blosc2.string(max_length=4, null_storage="mask"))), + ("ts", object, blosc2.field(blosc2.timestamp(null_storage="mask"))), + ], + ) + t = blosc2.CTable(Row, expected_size=16) + t.extend( + [ + (1, "a", "p", datetime.datetime(2020, 1, 1)), + (None, None, None, None), + (3, "c", "r", datetime.datetime(2022, 1, 1)), + ] + ) + return t + + +def test_a_row_shows_none_for_a_null_not_the_fill(): + """The fill is explicitly not part of the format contract. + + It was observable here and indistinguishable from data -- and inconsistent + within a single row, since the ``vlstring`` beside it already read ``None`` + for the same missing cell. + """ + row = mixed_null_table()[1] + assert (row.n, row.v, row.t, row.ts) == (None, None, None, None) + + +def test_iteration_and_repr_agree_with_indexing(): + t = mixed_null_table() + assert [r.n for r in t] == [1, None, 3] + assert "None" in repr(t) + + +def test_row_reads_stay_aligned_after_a_deletion(): + t = mixed_null_table() + t.delete(0) + assert t[0].n is None + assert [r.n for r in t] == [None, 3] + + +def test_a_sentinel_column_still_shows_its_sentinel_in_a_row(): + """There the stand-in *is* the contract: the user chose it and writes it. + + Substituting would change long-standing behaviour rather than hide an + implementation detail; ``is_null()`` is the uniform test across both. + """ + Row = dataclasses.make_dataclass( + "SentRow", [("n", int, blosc2.field(blosc2.int64(nullable=True, null_value=-1)))] + ) + t = blosc2.CTable(Row, expected_size=8) + t.extend([(1,), (-1,), (3,)]) + assert t[1].n == -1 + assert t["n"].is_null().tolist() == [False, True, False] From 50bce2b3cf55c726df716bba4ff552fa9411d166 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 16:50:03 +0200 Subject: [PATCH 43/47] Record the second round of review findings Five more defects and their fixes, in the plan beside the first two rounds and in the 4.11.0 bug-fix list. Co-Authored-By: Claude Fable 5 --- RELEASE_NOTES.md | 16 ++++++++++++++++ plans/mask-based-nulls.md | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9c0091879..847bb2293 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -201,6 +201,22 @@ and has its own spelling for nulls (`None` among the values). looked for nulls *inside* a value that was a single cell rather than a batch. Scalar broadcast works again, and `col[0:2] = None` now makes every selected row null. +- **`extend()` from another table lost nulls between storages.** Copying rows + from a mask-backed column into a sentinel-backed one (or the reverse, or + between two sentinels reserving different values) wrote whatever stood in for + the null as real data. Nullity is translated now. +- **A row read showed a mask column's fill instead of `None`.** `t[i]`, + iteration and `repr` surfaced the placeholder that occupies a null slot, which + is not part of the format contract — and disagreed with a `vlstring` column in + the same row, which already read `None`. Sentinel columns still show their + sentinel, which is the value you chose. +- **`to_numpy(masked=True)` and `dropna()` raised on a nullable dictionary + column**, which reported its nulls per physical slot rather than per live row. +- **A null in a complex column reached pandas as `nan+0j`** rather than as + missing. +- **A nullable `ndarray` of `bool` forgot it had been widened when reopened**, so + converting it to mask storage left it `uint8`, and the guard against a + dtype-changing in-place conversion on a persistent table stopped firing. - **`convert_nulls(to="sentinel")` refused over a value in a deleted row.** The collision check scanned physical slots, so a proposed sentinel present only in a row already deleted — unreadable, and dropped by the next `compact()` — diff --git a/plans/mask-based-nulls.md b/plans/mask-based-nulls.md index fae296b33..19b388954 100644 --- a/plans/mask-based-nulls.md +++ b/plans/mask-based-nulls.md @@ -1224,6 +1224,41 @@ further defects, **all reproduced against the tree and none fixed yet**, ordered > checked against a NumPy SQL oracle over 24 combinations — {mask, sentinel} × {indexed, > unindexed} × 6 expression shapes including `|` and the string form — all agreeing exactly. +### Third pass — the sweep re-run after the fixes (2026-08-10) + +Five more, all reproduced and all fixed the same day: + +7. **`extend()` from another table did not translate nullity.** It carried a validity sidecar + across only when the *source* used mask storage, so any pair of tables that spelled nulls + differently lost them outright. Mask into sentinel wrote the fill as real data; sentinel into + mask wrote the reserved value as real data; two sentinel columns with different reserved values + had the same problem. Fixed by reading the source's nullity through its own channel and writing + it the way the destination spells it, replacing the stand-in either way — which also has to be + *storable*, since a source sentinel string can exceed the destination's `max_length` and would + fail validation. The substitution promotes the dtype rather than casting to either side's: + writing `__BLOSC2_NULL__` into a `U4` truncated it to `__BL`, and casting the other way would + truncate real values the wider column was entitled to hold. Verified over 24 combinations. + +8. **A row read showed the fill.** `t[i]`, iteration and `repr` read the values array and nothing + else, so a mask null surfaced as `0` / `""` / epoch `NaT` — the one thing the fill must never be, + and inconsistent inside a single row beside a `vlstring` already reading `None`. Both row paths + (`_rows_to_dicts`, `_physical_row_value`) needed it. **Sentinel columns are deliberately left + alone**: there the stand-in *is* the contract. + +9. **`NDArraySpec.bool_widened_to_uint8` was never serialized.** `dtype_str` stores the *already + widened* `uint8`, byte-identical to a declared `uint8` column, so a reopened column looked + declared: `_unflip_mask_bool_dtype` stopped firing and `_convert_changes_dtype` reported no + change, disabling the guard that refuses a dtype-changing in-place conversion on a persistent + table. The flag is now written, and only when set. + +10. **A dictionary column answered `null_mask()` per physical slot**, where the contract — and every + other kind — is one flag per live row. `to_numpy(masked=True)` raised `MaskError` and `dropna()` + raised "shape mismatch", the latter since long before this branch. + +11. **A complex null reached pandas as `nan+0j`.** The object-cell branch returned a Python list, and + pandas re-inferred `complex128` from it, folding the `None` back into a value indistinguishable + from a genuine NaN. Returning an object array pins it. + Checked and reported clean by the sweep: the Kleene channels and their reflected operators, the masked min/max row-extremum substitution, the `FLAG_HAS_NAN` bail, `_sentinel_can_match`'s guard elision, CSV `\N`/`\E` round-trip, Arrow/persistence/`extend`-from-table round-trips, From f707af8a32c069a417bd50b2cd47654d5c4171ad Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 23:23:30 +0200 Subject: [PATCH 44/47] Stop a memory-scaling assertion failing on unrelated allocations test_constructors_string_dtype_do_not_materialize_a_fill_list measures peak memory for two array sizes and asserts the difference stays small, which is the right shape of test: a materialized fill list scales with shape[0] where the streamed build is flat. The bound was too tight to survive company. tracemalloc reports the peak of everything the process allocates while it runs, so garbage from earlier tests lands in the measurement. The test passed alone and in its own module but failed in a full *serial* run under NumPy 2.2 -- which is what Pyodide ships -- at 6.8 MiB against a 4.0 MiB bound. Measured in isolation the figure is ~0.0 MiB on both 2.2.6 and 2.5.1, so nothing was actually materializing a list; the peak was someone else's. Collecting before each measurement removes most of it, and the bound moves to 12 MiB, which is still far below the ~27 MiB a real fill list adds -- the signal this is looking for scales with shape[0], and noise does not. Found while chasing three CI failures on this branch, of which the two WASM/Pyodide ones are this test: Pyodide runs NumPy 2.2.5 in one process, serially, which is exactly the combination that reproduces it. The ~50 tests added by the review fixes shifted allocation patterns enough to tip a marginal assertion that had been passing. Verified: full serial suite under NumPy 2.2.6 (8967 passed), full suite on 2.5.1 (9309 passed). Co-Authored-By: Claude Fable 5 --- tests/ctable/test_utf8.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index e0d7bfa29..23d6fc3ab 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -2311,9 +2311,14 @@ def test_constructors_string_dtype_do_not_materialize_a_fill_list(): cancels the platform's baseline allocation, which is several MiB on Windows and half that elsewhere -- a fixed bound only tracks that baseline. """ + import gc import tracemalloc def peak_for(n): + # Collect first: tracemalloc reports the peak of everything the process + # allocates while it is running, so garbage left by earlier tests lands + # in this measurement and has nothing to do with the fill. + gc.collect() tracemalloc.start() try: arr = blosc2.zeros(n, dtype=STRING_DTYPE) @@ -2330,8 +2335,15 @@ def peak_for(n): # these two sizes, where the streamed build does not move at all. Both sizes # are well past the point where the chunk size stops growing, so the transient # buffer -- the whole of the peak -- is the same for each. + # + # The bound is well below that 27 MiB but well above the streamed build's + # own figure, which measures ~0 MiB in isolation on every NumPy tested: a + # tighter one turned this into an order-dependent failure, passing alone and + # failing in a full serial run under NumPy 2.2 (which is what Pyodide ships) + # as unrelated allocations moved the peak around. What is being detected + # here scales with shape[0]; noise does not. grown = (large - small) / 2**20 - assert grown < 4.0, f"peak grew {grown:.1f} MiB with 8x the rows: a fill list was materialized" + assert grown < 12.0, f"peak grew {grown:.1f} MiB with 8x the rows: a fill list was materialized" def test_utf8_dispatch_round_trips_conversion(): From fc76b49fddc5acde1ba8f7282a74ab147e071066 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 23:36:46 +0200 Subject: [PATCH 45/47] Wait for dim mode instead of asserting it instantly test_3d_dim_mode_fixed_value failed on the macOS 3.12 runner at `assert app._dim_mode`, immediately after `pilot.press("d")`. The app settles that state on a later frame, so the assertion is a race: it wins on an idle machine and loses on a loaded runner. The very next line in the same test already awaits wait_for_table before asserting, which is the idiom the rest of the module uses 60 times over. wait_for_dim_mode mirrors wait_for_table -- pause until the state is what was asked for, or fail after 100 frames -- and replaces the four instantaneous assertions, two entering dim mode and two leaving it. It still fails if the mode never toggles, verified by giving the loop zero iterations, so nothing is papered over. Nothing on this branch touches b2view; the test is TUI-only and pytest.ini excludes `tui` by default, which is why local runs never saw it. build.yml overrides `-m`, so CI does run them -- worth knowing, since it means a whole class of test is invisible to the default local command. Verified: three consecutive serial runs and one parallel run, all 15 passing. Co-Authored-By: Claude Fable 5 --- tests/b2view/test_basics.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/b2view/test_basics.py b/tests/b2view/test_basics.py index f4c12a41a..570322b9f 100644 --- a/tests/b2view/test_basics.py +++ b/tests/b2view/test_basics.py @@ -96,6 +96,20 @@ async def wait_for_table(pilot) -> None: raise AssertionError("data table never finished loading") +async def wait_for_dim_mode(pilot, expected: bool) -> None: + """Wait until dim mode has actually toggled. + + ``pilot.press`` delivers the key, but the app settles its own state on a + later frame -- so asserting straight after the press is a race that a loaded + CI runner loses. Same shape as :func:`wait_for_table`, and the same reason. + """ + for _ in range(100): + await pilot.pause() + if pilot.app._dim_mode is expected: + return + raise AssertionError(f"dim mode never became {expected}") + + async def wait_until(pilot, predicate, *, message="condition not met in time") -> None: """Pump the event loop until *predicate* holds. @@ -345,12 +359,12 @@ async def test_2d_paging(store_path): # In dim mode the active (row) dim scrolls by one row, nudging the # window off the page grid. await pilot.press("d") - assert app._dim_mode + await wait_for_dim_mode(pilot, True) await pilot.press("up") await wait_for_table(pilot) assert app.table_page["start"] == 1 # off-grid by one row await pilot.press("escape") - assert not app._dim_mode + await wait_for_dim_mode(pilot, False) # An explicit page down now snaps back onto the page grid instead of # carrying the one-row offset (the bug), and page up returns to 0. @@ -379,7 +393,7 @@ async def test_3d_dim_mode_fixed_value(store_path): assert layout.navigable_dims == [1, 2] await pilot.press("d") # enter dim mode (active dim is d0, fixed) - assert app._dim_mode + await wait_for_dim_mode(pilot, True) await pilot.press("up") # d0: 0 -> 1 await wait_for_table(pilot) @@ -391,7 +405,7 @@ async def test_3d_dim_mode_fixed_value(store_path): np.testing.assert_allclose(page["data"][str(c)], expected[page["start"] : page["stop"], c]) await pilot.press("escape") - assert not app._dim_mode + await wait_for_dim_mode(pilot, False) # ── CTable: row paging, goto, and wide tables ──────────────────────────── From 15e5574a59880b3025293c35ef63aef2f88ebe5f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 10 Aug 2026 23:46:10 +0200 Subject: [PATCH 46/47] Derive the fill-list bound from the pointer size, and skip it on WASM The CI run that finally named this test measured 4.2 MiB against a 4.0 MiB bound under Pyodide -- the margin was never real, and my first fix (a flat 12 MiB) guessed in the wrong direction for that platform. What the test detects is one pointer per row, and a pointer is not the same size everywhere: 8 bytes on the 64-bit runners, 4 under wasm32. So materializing the fill costs ~27 MiB here and only ~13 MiB on Pyodide, and a fixed bound cannot sit sensibly between the signal and the noise on both. The bound is now derived -- half the pointer-list cost for the platform running the test -- which keeps a real regression caught everywhere, verified by allocating a list inside the measured region: 26.7 MiB, failed as intended. Pyodide is skipped outright. Its noise floor (~4 MiB, from allocations this test does not control) is close enough to its own 13 MiB signal that the measurement cannot separate them, and the property being guarded is architecture-independent Python: if zeros(StringDType) does not build a list on x86-64 it does not build one on wasm32, since it is the same code path. What would be lost is the measurement, not the coverage -- and the repo already scopes tests this way where the environment cannot support them. Co-Authored-By: Claude Fable 5 --- tests/ctable/test_utf8.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/ctable/test_utf8.py b/tests/ctable/test_utf8.py index 23d6fc3ab..d4b6af359 100644 --- a/tests/ctable/test_utf8.py +++ b/tests/ctable/test_utf8.py @@ -2299,6 +2299,12 @@ def test_constructors_string_dtype_reject_nd(): blosc2.zeros(3, dtype=STRING_DTYPE, urlpath="unused.b2nd") +@pytest.mark.skipif( + blosc2.IS_WASM, + reason="peak-memory scaling is not measurable under Pyodide: its noise floor " + "(~4 MiB observed) is close to the 4-byte-pointer signal this looks for, and " + "the property itself is architecture-independent", +) def test_constructors_string_dtype_do_not_materialize_a_fill_list(): """The fill is one string repeated; building a list of it is pure overhead. @@ -2331,19 +2337,25 @@ def peak_for(n): small = peak_for(500_000) large = peak_for(4_000_000) - # 8x the rows is 8x the pointer list: materializing one adds ~27 MiB between - # these two sizes, where the streamed build does not move at all. Both sizes - # are well past the point where the chunk size stops growing, so the transient - # buffer -- the whole of the peak -- is the same for each. + # 8x the rows is 8x the pointer list, where the streamed build does not move + # at all. Both sizes are well past the point where the chunk size stops + # growing, so the transient buffer -- the whole of the peak -- is the same + # for each. # - # The bound is well below that 27 MiB but well above the streamed build's - # own figure, which measures ~0 MiB in isolation on every NumPy tested: a - # tighter one turned this into an order-dependent failure, passing alone and - # failing in a full serial run under NumPy 2.2 (which is what Pyodide ships) + # The bound is derived rather than guessed, because the thing being detected + # is one pointer per row and a pointer is not the same size everywhere: 8 + # bytes here, 4 on wasm32, so materializing costs ~27 MiB on a 64-bit box + # and ~13 MiB under Pyodide. Half of that keeps a real regression caught on + # both while leaving room for the noise floor, which is what a fixed bound + # kept tripping over -- this passed alone and failed in a full serial run, # as unrelated allocations moved the peak around. What is being detected - # here scales with shape[0]; noise does not. + # scales with shape[0]; noise does not. + pointer_list_mib = (4_000_000 - 500_000) * np.dtype(np.intp).itemsize / 2**20 grown = (large - small) / 2**20 - assert grown < 12.0, f"peak grew {grown:.1f} MiB with 8x the rows: a fill list was materialized" + assert grown < pointer_list_mib / 2, ( + f"peak grew {grown:.1f} MiB with 8x the rows (a materialized fill list " + f"would cost ~{pointer_list_mib:.0f} MiB here): a fill list was materialized" + ) def test_utf8_dispatch_round_trips_conversion(): From d742399f7e6b0f0b5e669f672bc9ede1b7bed140 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 11 Aug 2026 00:09:29 +0200 Subject: [PATCH 47/47] Wait for a Textual screen instead of pausing a fixed number of times test_group_key_applies_and_escape_clears failed on the Windows runner at `assert isinstance(pilot.app.screen, GroupByScreen)`, one pilot.pause() after the key that opens it. Same shape as the dim-mode race fixed a few commits back, and the same cause: pilot.press delivers the key, but the app pushes the screen over the frames that follow, so a fixed number of pauses is the race with a bigger constant rather than a fix. The `await pilot.pause()` doubled up before two of these assertions is what that workaround looks like in the source. tests/b2view/tui_wait.py holds the shared waits -- wait_until, and wait_for_screen for the transition itself, in both directions. Each b2view module had grown its own _wait_table-shaped helper and test_basics had a private wait_until; the screen races are identical across all of them, so this is the piece worth having in one place. test_basics now imports the shared wait_until rather than defining its own. 27 immediate assertions across the three suites become bounded waits: 22 screen transitions and 4 dismissals in test_basics, 8 in test_group, 1 in test_sort. They still fail if the screen never arrives -- verified by setting the frame budget to zero, which fails them with "screen never became GroupBarScreen" rather than passing vacuously. Nothing on this branch touches b2view. These are TUI-only tests, excluded by pytest.ini's default marker selection and re-enabled by build.yml's -m override, which is why they only ever fail in CI. Verified: three serial runs and one parallel run of the b2view suite, and the full CI selection at 9335 passed. Co-Authored-By: Claude Fable 5 --- tests/b2view/test_basics.py | 96 +++++++++++-------------------------- tests/b2view/test_group.py | 27 ++++------- tests/b2view/test_sort.py | 5 +- tests/b2view/tui_wait.py | 59 +++++++++++++++++++++++ 4 files changed, 99 insertions(+), 88 deletions(-) create mode 100644 tests/b2view/tui_wait.py diff --git a/tests/b2view/test_basics.py b/tests/b2view/test_basics.py index 570322b9f..2e8c340e0 100644 --- a/tests/b2view/test_basics.py +++ b/tests/b2view/test_basics.py @@ -47,6 +47,7 @@ import tree_store_gen as gen from textual.widgets import DataTable, Input, SelectionList, Tree +from tui_wait import wait_for_screen, wait_until from blosc2.b2view.app import ( B2ViewApp, @@ -110,20 +111,6 @@ async def wait_for_dim_mode(pilot, expected: bool) -> None: raise AssertionError(f"dim mode never became {expected}") -async def wait_until(pilot, predicate, *, message="condition not met in time") -> None: - """Pump the event loop until *predicate* holds. - - Setting ``Input.value`` posts an ``Input.Changed`` that rebuilds dependent widgets - asynchronously; a single ``pilot.pause()`` is not always enough on slower/loaded CI - (e.g. Windows), so poll until the resulting state settles. - """ - for _ in range(100): - await pilot.pause() - if predicate(): - return - raise AssertionError(message) - - async def focus_data_table(pilot) -> DataTable: table = pilot.app.query_one("#data-table", DataTable) table.focus() @@ -210,11 +197,9 @@ async def test_tree_and_panel_focus(store_path): # '?' opens the help screen; escape closes it await pilot.press("question_mark") - await pilot.pause() - assert isinstance(app.screen, HelpScreen) + await wait_for_screen(pilot, HelpScreen) await pilot.press("escape") - await pilot.pause() - assert not isinstance(app.screen, HelpScreen) + await wait_for_screen(pilot, HelpScreen, present=False) # ── 1-D array: row paging beyond the viewport ──────────────────────────── @@ -334,8 +319,7 @@ async def test_2d_paging(store_path): # modal pre-fills the current index and pre-selects it, so typing a new # index replaces it (not appends, e.g. "0" + "97" -> "097"). await pilot.press("c") - await pilot.pause() - assert isinstance(app.screen, GoToColumnScreen) + await wait_for_screen(pilot, GoToColumnScreen) gotocol_input = app.screen.query_one("#gotocol-input", Input) assert gotocol_input.value == "0" # pre-filled with current column for ch in "97": @@ -447,8 +431,7 @@ async def test_ctable_row_paging_and_goto(store_path): # 'g' opens the goto modal; submit a row in the middle await pilot.press("g") - await pilot.pause() - assert isinstance(app.screen, GoToRowScreen) + await wait_for_screen(pilot, GoToRowScreen) app.screen.query_one("#goto-input", Input).value = "250" await pilot.press("enter") await wait_for_table(pilot) @@ -553,8 +536,7 @@ async def test_ctable_column_paging(store_path): # 'c' opens a searchable column picker (type to filter, ↑/↓, Enter); # the row position is kept. Pick v12 by typing its name. await pilot.press("c") - await pilot.pause() - assert isinstance(app.screen, ColumnSelectScreen) + await wait_for_screen(pilot, ColumnSelectScreen) for ch in "v12": await pilot.press(ch) await pilot.pause() @@ -569,8 +551,7 @@ async def test_ctable_column_paging(store_path): # ↑/↓ drive the highlight: filter to the v1x family, then arrow to v12 await pilot.press("c") - await pilot.pause() - assert isinstance(app.screen, ColumnSelectScreen) + await wait_for_screen(pilot, ColumnSelectScreen) for ch in "v1": # matches v10, v11, ..., v19 in column order await pilot.press(ch) await pilot.pause() @@ -583,8 +564,7 @@ async def test_ctable_column_paging(store_path): # escape cancels the picker without moving await pilot.press("c") - await pilot.pause() - assert isinstance(app.screen, ColumnSelectScreen) + await wait_for_screen(pilot, ColumnSelectScreen) await pilot.press("escape") await wait_for_table(pilot) assert app.table_page["col_start"] == all_names.index("v12") @@ -624,8 +604,7 @@ async def test_ctable_filtering(store_path): async def submit_filter(expr: str) -> None: await pilot.press("f") - await pilot.pause() - assert isinstance(app.screen, FilterScreen) + await wait_for_screen(pilot, FilterScreen) app.screen.query_one("#filter-input", Input).value = expr await pilot.press("enter") await wait_for_table(pilot) @@ -675,8 +654,7 @@ async def submit_filter(expr: str) -> None: # '/' opens the picker with the currently-shown columns pre-checked. await pilot.press("slash") - await pilot.pause() - assert isinstance(app.screen, ColumnFilterScreen) + await wait_for_screen(pilot, ColumnFilterScreen) sel = app.screen.query_one("#colfilter-list", SelectionList) assert sel.option_count == ncols assert len(sel.selected) == ncols # all checked initially @@ -706,8 +684,7 @@ async def submit_filter(expr: str) -> None: # The goto-column picker lists names within the visible set. await pilot.press("c") - await pilot.pause() - assert isinstance(app.screen, ColumnSelectScreen) + await wait_for_screen(pilot, ColumnSelectScreen) for ch in "v15": await pilot.press(ch) await pilot.pause() @@ -726,8 +703,7 @@ async def submit_filter(expr: str) -> None: # Re-opening pre-checks the visible set; Escape cancels (no change). await pilot.press("slash") - await pilot.pause() - assert isinstance(app.screen, ColumnFilterScreen) + await wait_for_screen(pilot, ColumnFilterScreen) assert len(app.screen.query_one("#colfilter-list", SelectionList).selected) == ncols - 1 await pilot.press("escape") await wait_for_table(pilot) @@ -759,8 +735,7 @@ async def test_plot_column(store_path): await focus_data_table(pilot) await pilot.press("p") - await pilot.pause() - assert isinstance(app.screen, PlotScreen) + await wait_for_screen(pilot, PlotScreen) screen = app.screen # Bucketed envelope covering the whole leaf; bracketed by true extremes @@ -806,12 +781,10 @@ async def test_plot_column(store_path): # 'g' opens a range modal; an exact range zooms there and reads it exactly. await pilot.press("g") - await pilot.pause() - assert isinstance(app.screen, PlotRangeScreen) + await wait_for_screen(pilot, PlotRangeScreen) app.screen.query_one("#range-input", Input).value = "1000:2000" await pilot.press("enter") - await pilot.pause() - assert isinstance(app.screen, PlotScreen) + await wait_for_screen(pilot, PlotScreen) screen = app.screen assert (screen.row_start, screen.row_stop) == (1000, 2000) sub = leaf1_values()[1000:2000] @@ -849,11 +822,9 @@ async def test_plot_column(store_path): # 'p' re-opens the plot; 'escape' is the only way to close it await pilot.press("p") - await pilot.pause() - assert isinstance(app.screen, PlotScreen) + await wait_for_screen(pilot, PlotScreen) await pilot.press("escape") - await pilot.pause() - assert not isinstance(app.screen, PlotScreen) + await wait_for_screen(pilot, PlotScreen, present=False) async def test_plot_view_locks_ctable_window(store_path): @@ -878,8 +849,7 @@ async def test_plot_view_locks_ctable_window(store_path): # Plot column 'b' (== row index), then zoom to an exact 100:110 range. table.move_cursor(column=app.table_page["columns"].index("b")) await pilot.press("p") - await pilot.pause() - assert isinstance(app.screen, PlotScreen) + await wait_for_screen(pilot, PlotScreen) await pilot.press("g") await pilot.pause() app.screen.query_one("#range-input", Input).value = "100:110" @@ -934,15 +904,13 @@ async def test_plot_hires_view(store_path): table.move_cursor(column=app.table_page["columns"].index("b")) await pilot.press("p") - await pilot.pause() - assert isinstance(app.screen, PlotScreen) + await wait_for_screen(pilot, PlotScreen) plot = app.screen # 'h' opens the hi-res view in envelope mode over the whole range — no # zoom gate; it reuses the on-screen envelope, so it always renders. await pilot.press("h") - await pilot.pause() - assert isinstance(app.screen, HiResPlotScreen) + await wait_for_screen(pilot, HiResPlotScreen) hires = app.screen assert hires._mode == "envelope" assert "min/max envelope" in hires._current_title() @@ -986,8 +954,7 @@ async def test_plot_scatter_col_vs_col(store_path): # Plot column 'b' (== row index), then zoom to a small range. table.move_cursor(column=app.table_page["columns"].index("b")) await pilot.press("p") - await pilot.pause() - assert isinstance(app.screen, PlotScreen) + await wait_for_screen(pilot, PlotScreen) plot = app.screen await pilot.press("g") await pilot.pause() @@ -998,8 +965,7 @@ async def test_plot_scatter_col_vs_col(store_path): # 's' opens the searchable Y-column picker over all visible columns. await pilot.press("s") - await pilot.pause() - assert isinstance(app.screen, ColumnSelectScreen) + await wait_for_screen(pilot, ColumnSelectScreen) from textual.widgets import OptionList picker = app.screen @@ -1019,8 +985,7 @@ async def test_plot_scatter_col_vs_col(store_path): picker.query_one("#colselect-input", Input).value = "c" await pilot.pause() # let the live filter repopulate the list await pilot.press("enter") - await pilot.pause() - assert isinstance(app.screen, ScatterPlotScreen) + await wait_for_screen(pilot, ScatterPlotScreen) scatter = app.screen # Row-aligned over the framed range: b == row index, c == row * 1.5. assert scatter.xcol == "b" @@ -1035,8 +1000,7 @@ async def test_plot_scatter_col_vs_col(store_path): from blosc2.b2view.app import HiResPlotScreen, TextualImage await pilot.press("h") - await pilot.pause() - assert isinstance(app.screen, HiResPlotScreen) + await wait_for_screen(pilot, HiResPlotScreen) assert app.screen.query_one("#hires-image", TextualImage) is not None await pilot.press("escape") await pilot.pause() @@ -1079,21 +1043,18 @@ class TaggedRow: # Enter on the cheap 'id' column does nothing special (no modal). table.move_cursor(row=2, column=app.table_page["columns"].index("id")) await pilot.press("enter") - await pilot.pause() - assert not isinstance(app.screen, CellDetailScreen) + await wait_for_screen(pilot, CellDetailScreen, present=False) # Enter on the skipped 'tags' cell decodes just that row into a modal. table.move_cursor(row=2, column=app.table_page["columns"].index("tags")) await pilot.press("enter") - await pilot.pause() - assert isinstance(app.screen, CellDetailScreen) + await wait_for_screen(pilot, CellDetailScreen) assert app.screen._value == [0, 1, 2] # row 2 of the generator above assert app.screen._row == 2 # esc returns to the table with its position intact. await pilot.press("escape") - await pilot.pause() - assert not isinstance(app.screen, CellDetailScreen) + await wait_for_screen(pilot, CellDetailScreen, present=False) assert table.cursor_row == 2 @@ -1183,10 +1144,9 @@ def fake_download(url, dst, on_progress): info_url="https://cat2.cloud/demo/api/info/@public/large/fetched.b2z", ) async with app.run_test(size=TERM_SIZE) as pilot: - await pilot.pause() # The bundle is not opened yet: the download screen is up, and the info # endpoint's size made the progress bar determinate. - assert isinstance(app.screen, DownloadScreen) + await wait_for_screen(pilot, DownloadScreen) assert app.browser is None assert app.screen.query_one("#download-bar", ProgressBar).total == size diff --git a/tests/b2view/test_group.py b/tests/b2view/test_group.py index 8affbd314..bb225f298 100644 --- a/tests/b2view/test_group.py +++ b/tests/b2view/test_group.py @@ -18,6 +18,7 @@ import numpy as np import pytest +from tui_wait import wait_for_screen import blosc2 @@ -251,8 +252,7 @@ async def test_group_key_applies_and_escape_clears(group_store): await pilot.pause() await pilot.press("G") - await pilot.pause() - assert isinstance(pilot.app.screen, GroupByScreen) + await wait_for_screen(pilot, GroupByScreen) await pilot.press("enter") # key list -> operation list await pilot.press("enter") # "count rows" (first op) applies, no value col await _wait_table(pilot) @@ -277,8 +277,7 @@ async def test_group_key_applies_and_escape_clears(group_store): # Sort the grouped result by one of its columns via 'S'. await pilot.press("S") - await pilot.pause() - assert isinstance(pilot.app.screen, SortByScreen) + await wait_for_screen(pilot, SortByScreen) await pilot.press("enter") # apply the highlighted column await _wait_table(pilot) assert pilot.app.browser.get_group_sort("/ctable") is not None @@ -362,8 +361,7 @@ async def test_group_config_cached_and_reused(group_store): pilot.app.query_one("#data-table", DataTable).focus() await pilot.pause() await pilot.press("G") - await pilot.pause() - assert isinstance(pilot.app.screen, GroupByScreen) + await wait_for_screen(pilot, GroupByScreen) assert pilot.app.screen._current == ("region", "mean", "amount") @@ -388,19 +386,15 @@ async def test_group_bar_chart_and_hires(group_store): await pilot.press("down", "enter") # value: amount -> apply await _wait_table(pilot) await pilot.press("p") - await pilot.pause() - assert isinstance(pilot.app.screen, GroupBarScreen) + await wait_for_screen(pilot, GroupBarScreen) assert pilot.app.screen.numeric is False # 'h' opens the hi-res matplotlib bar chart; esc returns to the plotext bars. await pilot.press("h") - await pilot.pause() - await pilot.pause() - assert isinstance(pilot.app.screen, HiResPlotScreen) + await wait_for_screen(pilot, HiResPlotScreen) assert pilot.app.screen._mode == "bar" await pilot.press("escape") - await pilot.pause() - assert isinstance(pilot.app.screen, GroupBarScreen) + await wait_for_screen(pilot, GroupBarScreen) @pytest.mark.asyncio @@ -424,12 +418,9 @@ async def test_group_numeric_key_plots_as_line(group_store): await pilot.press("down", "enter") # value: amount -> apply await _wait_table(pilot) await pilot.press("p") - await pilot.pause() - assert isinstance(pilot.app.screen, GroupBarScreen) + await wait_for_screen(pilot, GroupBarScreen) assert pilot.app.screen.numeric is True await pilot.press("h") # hi-res is a stem/impulse plot, not bars - await pilot.pause() - await pilot.pause() - assert isinstance(pilot.app.screen, HiResPlotScreen) + await wait_for_screen(pilot, HiResPlotScreen) assert pilot.app.screen._mode == "stem" diff --git a/tests/b2view/test_sort.py b/tests/b2view/test_sort.py index ba721e321..2988087cc 100644 --- a/tests/b2view/test_sort.py +++ b/tests/b2view/test_sort.py @@ -26,6 +26,8 @@ if blosc2.IS_WASM: pytest.skip("Textual apps need a terminal driver (termios)", allow_module_level=True) +from tui_wait import wait_for_screen + from blosc2.b2view.app import B2ViewApp, SortByScreen from blosc2.b2view.model import StoreBrowser @@ -195,8 +197,7 @@ async def test_sort_key_applies_and_escape_clears(sort_store): # 'S' opens the sort dropdown listing the FULL-indexed columns. await pilot.press("S") - await pilot.pause() - assert isinstance(app.screen, SortByScreen) + await wait_for_screen(pilot, SortByScreen) # Enter applies the highlighted column ascending; grid reorders. await pilot.press("enter") diff --git a/tests/b2view/tui_wait.py b/tests/b2view/tui_wait.py new file mode 100644 index 000000000..e1922deae --- /dev/null +++ b/tests/b2view/tui_wait.py @@ -0,0 +1,59 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +"""Waiting for a Textual app to settle, shared by the b2view suites. + +``pilot.press`` delivers the key, but the app acts on it over the frames that +follow: pushing a screen, rebuilding a widget, reloading a page. Asserting +straight after the press is therefore a race, and one that an idle developer +machine wins and a loaded CI runner loses -- which is how it reaches us, as a +single test failing on one platform. + +A fixed number of ``pilot.pause()`` calls is the same race with a bigger +constant; the ``await pilot.pause()`` doubled up before some assertions here is +what that looks like in the source. Poll for the state instead, with a bound +so a genuine hang still fails rather than hanging the suite. + +Each b2view module had grown its own ``_wait_table``-shaped helper; this is the +piece worth having in one place, since the races are identical across them. +""" + +from __future__ import annotations + +#: How many frames to pump before giving up. Generous: the cost of a high +#: bound is paid only when something is genuinely broken, while the cost of a +#: low one is a flake on whichever runner happens to be slowest that day. +MAX_FRAMES = 100 + + +async def wait_until(pilot, predicate, *, message="condition not met in time") -> None: + """Pump the event loop until *predicate* holds, or fail.""" + for _ in range(MAX_FRAMES): + await pilot.pause() + if predicate(): + return + raise AssertionError(message) + + +async def wait_for_screen(pilot, screen_cls, *, present: bool = True): + """Wait until the app's active screen is (or is no longer) *screen_cls*. + + Returns the active screen, so a caller that goes on to query widgets on it + reads the same object the wait settled on. + """ + what = screen_cls.__name__ + + def settled(): + return isinstance(pilot.app.screen, screen_cls) is present + + await wait_until( + pilot, + settled, + message=(f"screen never became {what}" if present else f"screen never left {what}") + + f" (still {type(pilot.app.screen).__name__})", + ) + return pilot.app.screen