From f628d178edcf8ac7f730709d6fbdf1cfb4a2a2f2 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 20:15:51 +0000 Subject: [PATCH 1/5] docs: add design spec for timestamp/duration leaf data hashing (ITL-438) Co-Authored-By: Claude Sonnet 4.6 --- ...06-28-timestamp-duration-hashing-design.md | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md diff --git a/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md b/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md new file mode 100644 index 0000000..c9b9004 --- /dev/null +++ b/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md @@ -0,0 +1,162 @@ +# Timestamp and Duration Type Support in Leaf Data Hashing + +**Date:** 2026-06-28 +**Issue:** ITL-438 +**Repos:** nauticalab/starfix (Rust), nauticalab/starfix-python (Python) + +--- + +## Overview + +`ArrowDigester` panics (Rust `todo!()`) or raises `NotImplementedError` (Python) when +hashing any Arrow table or array containing a `timestamp` or `duration` column. This +blocks any project that uses `datetime.datetime` values as data or tag values — a very +common pattern. The fix is a minimal two-line addition in each repo. + +--- + +## Goals & Success Criteria + +- `hash_table`, `hash_record_batch`, `hash_array`, and streaming `update` all work for + `timestamp` (all 4 units × tz-aware and tz-naive) and `duration` (all 4 units) +- Hashes are byte-for-byte identical between the Rust and Python implementations +- Different timestamp units produce different hashes (unit is captured in schema hash) +- Timezone-aware and timezone-naive timestamps with identical data produce different hashes + (tz is captured in schema hash) +- `interval` types remain unimplemented (out of scope) + +--- + +## Scope & Boundaries + +**In scope:** +- `timestamp[s/ms/us/ns]` with and without timezone +- `duration[s/ms/us/ns]` +- Raw buffer layout tests (nullable bytes) in Rust +- Golden parity tests in Python (`test_golden_parity.py`) +- Schema serialization tests for the new types +- Updating the comprehensive `schema()` golden hash in Rust + +**Out of scope:** +- `interval` types (remain `todo!()`) +- Any other currently-unimplemented Arrow types +- Changes to `_data_type_to_value` / schema hashing (already correct) + +--- + +## Root Cause + +In `arrow_digester_core.rs`, the `array_digest_update` match has: + +```rust +DataType::Timestamp(_, _) => todo!(), +DataType::Duration(_) => todo!(), +``` + +In `arrow_digester.py`, `_element_size_for_type` has no handler for +`pa.types.is_timestamp` or `pa.types.is_duration`. + +The schema-hashing path (`_data_type_to_value` / `data_type_to_value`) already handles +both types correctly — only the leaf data hashing step is missing. + +--- + +## Arrow Storage Layout + +All `timestamp` and `duration` variants are stored as signed 64-bit integers (8 bytes per +element) in Arrow's physical layout. The unit (`s`, `ms`, `us`, `ns`) and timezone are +schema-level metadata, not data-buffer metadata. This is the same physical layout as +`time64`, `int64`, `date64`, etc. + +--- + +## Design + +### Rust fix (`src/arrow_digester_core.rs`) + +Replace the two `todo!()` arms with standalone calls, keeping them adjacent to the +existing `Time32`/`Time64` arms for readability: + +```rust +DataType::Timestamp(_, _) => Self::hash_fixed_size_array(effective_array, digest, 8), +// ↑ int64 physical storage; unit and tz are schema metadata, not data bytes + +DataType::Time32(_) => Self::hash_fixed_size_array(effective_array, digest, 4), +DataType::Time64(_) => Self::hash_fixed_size_array(effective_array, digest, 8), + +DataType::Duration(_) => Self::hash_fixed_size_array(effective_array, digest, 8), +// ↑ int64 physical storage; unit is schema metadata, not data bytes +``` + +### Python fix (`src/starfix/arrow_digester.py`) + +In `_element_size_for_type`, add two checks after the existing `time32`/`time64` block: + +```python +if pa.types.is_time32(dt): + return 4 +if pa.types.is_time64(dt): + return 8 +if pa.types.is_timestamp(dt): + return 8 # int64 physical storage; unit/tz are schema metadata +if pa.types.is_duration(dt): + return 8 # int64 physical storage; unit is schema metadata +``` + +No other files require changes. + +--- + +## Test Strategy + +### TDD sequence (required by starfix CLAUDE.md) + +1. Write failing Rust tests → confirm they panic from `todo!()` +2. Implement Rust fix → confirm all Rust tests pass +3. Capture golden hash values from Rust output +4. Write failing Python tests using those golden values → confirm `NotImplementedError` +5. Implement Python fix → confirm all Python tests pass + +### Rust tests (`tests/arrow_digester.rs`) + +| Test | Description | +|---|---| +| `timestamp_array_hashing` | All 4 units × tz-aware (`UTC`) and tz-naive — 8 `hash_array` assertions with golden hex | +| `duration_array_hashing` | All 4 units — 4 `hash_array` assertions with golden hex | +| `timestamp_units_differ` | Assert `timestamp[s]` ≠ `timestamp[ms]` ≠ `timestamp[us]` ≠ `timestamp[ns]` for same raw data | +| `timestamp_tz_differs` | Assert `timestamp[us, tz=UTC]` ≠ `timestamp[us]` for same raw data | +| `duration_units_differ` | Assert all 4 duration units differ for same raw data | +| `schema()` | Add timestamp and duration columns (all variants); update golden hash | + +### Rust raw buffer tests (`src/arrow_digester_core.rs`) + +| Test | Description | +|---|---| +| `digest_timestamp_nullable_bytes` | `TimestampMicrosecondArray [0, None, 3_600_000_000]` — verify null bits `[true, false, true]`, data digest = sha256 of two i64 LE values | +| `digest_duration_nullable_bytes` | `DurationMicrosecondArray [0, None, 3_600_000_000]` — same verification | + +### Python tests (`tests/test_golden_parity.py`) + +| Test | Description | +|---|---| +| `test_timestamp_utc_microsecond_array` | `timestamp[us, tz=UTC]` with 3 values (incl. null); assert hex matches Rust | +| `test_timestamp_naive_microsecond_array` | `timestamp[us]` with same values; assert hex ≠ tz-aware | +| `test_timestamp_all_units` | All 4 units × tz-aware and tz-naive; assert units differ | +| `test_duration_microsecond_array` | `duration[us]` with 3 values (incl. null); assert hex matches Rust | +| `test_duration_all_units` | All 4 duration units; assert units differ | + +### Python schema tests (`tests/test_arrow_digester.py`) + +| Test | Description | +|---|---| +| `test_timestamp_types_in_schema` | `_serialized_schema` produces `{"Timestamp":["Microsecond","UTC"]}` and `{"Timestamp":["Microsecond",null]}` | +| `test_duration_types_in_schema` | `_serialized_schema` produces `{"Duration":"Microsecond"}` etc. | + +--- + +## Implementation Sequence + +1. **Rust:** Write failing tests → implement fix → run `cargo test` → capture golden hashes → update `schema()` golden hash +2. **Python:** Write failing tests (using Rust golden values) → implement fix → run pytest +3. **Formatting:** Run `cargo fmt` before committing (required by CLAUDE.md) +4. **Two PRs:** One per repo, both targeting `dev`; Python PR notes dependency on Rust golden values From 32a8bc1b86c16c998d59ce74420d20c89927d3a8 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:06:45 +0000 Subject: [PATCH 2/5] docs: add Polars round-trip findings to timestamp/duration hashing spec (ITL-438) Co-Authored-By: Claude Sonnet 4.6 --- ...06-28-timestamp-duration-hashing-design.md | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md b/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md index c9b9004..460613e 100644 --- a/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md +++ b/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md @@ -70,6 +70,32 @@ schema-level metadata, not data-buffer metadata. This is the same physical layou --- +## Polars Round-Trip Behaviour + +Verified against polars 1.42.0 / pyarrow 24.0.0: + +| Type | Round-trip result | Stable? | +|---|---|---| +| `timestamp[ms/us/ns]` (any tz) | Type preserved verbatim | ✅ | +| `timestamp[us/ms/ns]` tz-naive | Type preserved verbatim | ✅ | +| `duration[ms/us/ns]` | Type preserved verbatim | ✅ | +| `timestamp[s, tz=*]` | **Coerced to `timestamp[ms, tz=*]`** | ❌ | +| `duration[s]` | **Coerced to `duration[ms]`** | ❌ | +| Polars native `datetime` | Always produces `timestamp[us, tz=UTC]` | ✅ | +| Timezone strings | Preserved verbatim (`UTC`, `America/New_York`, etc.) | ✅ | + +**Design decision:** starfix does **not** normalise `s`→`ms`. The hasher correctly +hashes whatever Arrow type it receives per the Arrow spec. `timestamp[s, value=1]` and +`timestamp[ms, value=1]` represent different points in time and must produce different +hashes. The `s`→`ms` coercion is a Polars limitation. + +**Practical implication for users:** Projects that pass Arrow data through Polars before +hashing should avoid `timestamp[s]` and `duration[s]` columns if hash stability across +the Polars boundary is required. Use `ms`, `us`, or `ns` instead — all three survive +Polars round-trips intact. + +--- + ## Design ### Rust fix (`src/arrow_digester_core.rs`) From 6e4daaeb79d2cbf9a9c9face8de7b1ab6d53ed1e Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:22:17 +0000 Subject: [PATCH 3/5] docs: target main branch in timestamp/duration hashing spec (ITL-438) Co-Authored-By: Claude Sonnet 4.6 --- .../specs/2026-06-28-timestamp-duration-hashing-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md b/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md index 460613e..1a28bb6 100644 --- a/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md +++ b/docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md @@ -185,4 +185,4 @@ No other files require changes. 1. **Rust:** Write failing tests → implement fix → run `cargo test` → capture golden hashes → update `schema()` golden hash 2. **Python:** Write failing tests (using Rust golden values) → implement fix → run pytest 3. **Formatting:** Run `cargo fmt` before committing (required by CLAUDE.md) -4. **Two PRs:** One per repo, both targeting `dev`; Python PR notes dependency on Rust golden values +4. **Two PRs:** One per repo, both targeting `main`; Python PR notes dependency on Rust golden values From 0e41acbf0ecfac4c01726eb56d1b044424344506 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:30:12 +0000 Subject: [PATCH 4/5] docs: add implementation plan for timestamp/duration hashing (ITL-438) Co-Authored-By: Claude Sonnet 4.6 --- .../2026-06-28-timestamp-duration-hashing.md | 1046 +++++++++++++++++ 1 file changed, 1046 insertions(+) create mode 100644 docs/metamorphic/plans/2026-06-28-timestamp-duration-hashing.md diff --git a/docs/metamorphic/plans/2026-06-28-timestamp-duration-hashing.md b/docs/metamorphic/plans/2026-06-28-timestamp-duration-hashing.md new file mode 100644 index 0000000..21da928 --- /dev/null +++ b/docs/metamorphic/plans/2026-06-28-timestamp-duration-hashing.md @@ -0,0 +1,1046 @@ +# Timestamp and Duration Leaf Data Hashing Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add support for `timestamp` and `duration` Arrow types in `ArrowDigester`'s leaf data hashing — replacing `todo!()` in Rust and a missing branch in Python — across both `nauticalab/starfix` and `nauticalab/starfix-python`. + +**Architecture:** All `timestamp` and `duration` Arrow variants are physically stored as signed 64-bit integers (8 bytes). The fix in both repos treats them identically to `time64`: a fixed-size 8-byte element. The Rust fix is implemented first; its test output provides the authoritative golden hash values that the Python golden-parity tests assert against. + +**Tech Stack:** Rust (arrow-rs, sha2, hex, pretty_assertions), Python (pyarrow ≥ 14, hashlib, pytest) + +--- + +**Working directories:** + +- Rust: `starfix/` (all `cargo` commands run here) +- Python: `starfix-python/` (all `pytest` commands run here) +- Branch in both repos: `eywalker/itl-438-add-support-for-timestamp-and-duration-types-in-leaf-data` + +--- + +## Phase 1: Rust (`nauticalab/starfix`) + +### Task 1: Add timestamp/duration imports to `tests/arrow_digester.rs` + +**Files:** +- Modify: `tests/arrow_digester.rs` (import block at top of `mod tests`) + +- [ ] **Step 1: Extend the existing arrow array import block** + +Open `tests/arrow_digester.rs`. Find the `use arrow::{ array::{` import block (lines 6–17). Replace it with: + +```rust +use arrow::{ + array::{ + ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array, Decimal32Array, + Decimal64Array, DictionaryArray, DurationMicrosecondArray, DurationMillisecondArray, + DurationNanosecondArray, DurationSecondArray, Float32Array, Float64Array, Int16Array, + Int32Array, Int64Array, Int8Array, LargeBinaryArray, LargeListArray, LargeListBuilder, + LargeStringArray, LargeStringBuilder, ListArray, ListBuilder, RecordBatch, StringArray, + StringBuilder, StructArray, Time32MillisecondArray, Time32SecondArray, + Time64MicrosecondArray, Time64NanosecondArray, TimestampMicrosecondArray, + TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, UInt16Array, + UInt32Array, UInt64Array, UInt8Array, + }, + datatypes::{Int32Type, Int8Type}, +}; +``` + +- [ ] **Step 2: Verify the file compiles** + +```bash +cargo check --tests 2>&1 | head -20 +``` + +Expected: no errors (warnings about unused imports are fine at this stage). + +--- + +### Task 2: Write failing unit tests for timestamp hashing + +**Files:** +- Modify: `tests/arrow_digester.rs` (add 5 new test functions after the existing `time_array_different_units_produce_different_hashes` test) + +- [ ] **Step 1: Add `timestamp_array_hashing` — prints hash values, asserts format only** + +Insert after `time_array_different_units_produce_different_hashes` (around line 192): + +```rust +#[test] +fn timestamp_array_hashing() { + // [0 epoch, None, 1000 units] — raw i64 values; meaning depends on unit + let values = vec![Some(0_i64), None, Some(1_000_i64)]; + + // tz-aware UTC, all 4 units + let h_s_utc = hex::encode(ArrowDigester::hash_array( + &TimestampSecondArray::from(values.clone()).with_timezone("UTC"), + )); + println!("ts_s_utc: {h_s_utc}"); + + let h_ms_utc = hex::encode(ArrowDigester::hash_array( + &TimestampMillisecondArray::from(values.clone()).with_timezone("UTC"), + )); + println!("ts_ms_utc: {h_ms_utc}"); + + let h_us_utc = hex::encode(ArrowDigester::hash_array( + &TimestampMicrosecondArray::from(values.clone()).with_timezone("UTC"), + )); + println!("ts_us_utc: {h_us_utc}"); + + let h_ns_utc = hex::encode(ArrowDigester::hash_array( + &TimestampNanosecondArray::from(values.clone()).with_timezone("UTC"), + )); + println!("ts_ns_utc: {h_ns_utc}"); + + // tz-naive, all 4 units + let h_s = hex::encode(ArrowDigester::hash_array( + &TimestampSecondArray::from(values.clone()), + )); + println!("ts_s: {h_s}"); + + let h_ms = hex::encode(ArrowDigester::hash_array( + &TimestampMillisecondArray::from(values.clone()), + )); + println!("ts_ms: {h_ms}"); + + let h_us = hex::encode(ArrowDigester::hash_array( + &TimestampMicrosecondArray::from(values.clone()), + )); + println!("ts_us: {h_us}"); + + let h_ns = hex::encode(ArrowDigester::hash_array( + &TimestampNanosecondArray::from(values.clone()), + )); + println!("ts_ns: {h_ns}"); + + // Weak assertions for now — exact values filled in after fix (Task 6) + assert!(h_s_utc.starts_with("000001"), "unexpected prefix: {h_s_utc}"); + assert!(h_ms_utc.starts_with("000001"), "unexpected prefix: {h_ms_utc}"); + assert!(h_us_utc.starts_with("000001"), "unexpected prefix: {h_us_utc}"); + assert!(h_ns_utc.starts_with("000001"), "unexpected prefix: {h_ns_utc}"); + assert!(h_s.starts_with("000001"), "unexpected prefix: {h_s}"); + assert!(h_ms.starts_with("000001"), "unexpected prefix: {h_ms}"); + assert!(h_us.starts_with("000001"), "unexpected prefix: {h_us}"); + assert!(h_ns.starts_with("000001"), "unexpected prefix: {h_ns}"); +} +``` + +- [ ] **Step 2: Add `duration_array_hashing`** + +```rust +#[test] +fn duration_array_hashing() { + let values = vec![Some(0_i64), None, Some(1_000_i64)]; + + let h_s = hex::encode(ArrowDigester::hash_array( + &DurationSecondArray::from(values.clone()), + )); + println!("dur_s: {h_s}"); + + let h_ms = hex::encode(ArrowDigester::hash_array( + &DurationMillisecondArray::from(values.clone()), + )); + println!("dur_ms: {h_ms}"); + + let h_us = hex::encode(ArrowDigester::hash_array( + &DurationMicrosecondArray::from(values.clone()), + )); + println!("dur_us: {h_us}"); + + let h_ns = hex::encode(ArrowDigester::hash_array( + &DurationNanosecondArray::from(values.clone()), + )); + println!("dur_ns: {h_ns}"); + + // Weak assertions for now — exact values filled in after fix (Task 6) + assert!(h_s.starts_with("000001"), "unexpected prefix: {h_s}"); + assert!(h_ms.starts_with("000001"), "unexpected prefix: {h_ms}"); + assert!(h_us.starts_with("000001"), "unexpected prefix: {h_us}"); + assert!(h_ns.starts_with("000001"), "unexpected prefix: {h_ns}"); +} +``` + +- [ ] **Step 3: Add `timestamp_units_differ`** + +```rust +#[test] +fn timestamp_units_differ() { + let values = vec![Some(1_000_i64), Some(2_000_i64)]; + let hashes = [ + hex::encode(ArrowDigester::hash_array( + &TimestampSecondArray::from(values.clone()), + )), + hex::encode(ArrowDigester::hash_array( + &TimestampMillisecondArray::from(values.clone()), + )), + hex::encode(ArrowDigester::hash_array( + &TimestampMicrosecondArray::from(values.clone()), + )), + hex::encode(ArrowDigester::hash_array( + &TimestampNanosecondArray::from(values.clone()), + )), + ]; + for i in 0..hashes.len() { + for j in (i + 1)..hashes.len() { + assert_ne!( + hashes[i], hashes[j], + "units {i} and {j} produced identical hashes" + ); + } + } +} +``` + +- [ ] **Step 4: Add `timestamp_tz_differs`** + +```rust +#[test] +fn timestamp_tz_differs() { + let values = vec![Some(1_000_i64), Some(2_000_i64)]; + let naive = hex::encode(ArrowDigester::hash_array( + &TimestampMicrosecondArray::from(values.clone()), + )); + let utc = hex::encode(ArrowDigester::hash_array( + &TimestampMicrosecondArray::from(values.clone()).with_timezone("UTC"), + )); + assert_ne!(naive, utc, "tz-naive and tz=UTC must produce different hashes"); +} +``` + +- [ ] **Step 5: Add `duration_units_differ`** + +```rust +#[test] +fn duration_units_differ() { + let values = vec![Some(1_000_i64), Some(2_000_i64)]; + let hashes = [ + hex::encode(ArrowDigester::hash_array( + &DurationSecondArray::from(values.clone()), + )), + hex::encode(ArrowDigester::hash_array( + &DurationMillisecondArray::from(values.clone()), + )), + hex::encode(ArrowDigester::hash_array( + &DurationMicrosecondArray::from(values.clone()), + )), + hex::encode(ArrowDigester::hash_array( + &DurationNanosecondArray::from(values.clone()), + )), + ]; + for i in 0..hashes.len() { + for j in (i + 1)..hashes.len() { + assert_ne!(hashes[i], hashes[j]); + } + } +} +``` + +- [ ] **Step 6: Run to confirm all 5 new tests fail** + +```bash +cargo test timestamp_array_hashing duration_array_hashing timestamp_units_differ timestamp_tz_differs duration_units_differ 2>&1 | tail -20 +``` + +Expected: all 5 tests fail with `not yet implemented` / `called \`Option::unwrap()\` on a \`None\` value` or similar panic from `todo!()`. + +--- + +### Task 3: Add timestamp/duration columns to the existing `schema()` test + +**Files:** +- Modify: `tests/arrow_digester.rs` (the `fn schema()` test — search for `"time64_nano"`) + +- [ ] **Step 1: Add 12 new fields to the schema definition** + +In `fn schema()`, find the line with `Field::new("time64_nano", DataType::Time64(TimeUnit::Nanosecond), false),` and insert immediately after it: + +```rust + // timestamp — all 4 units, tz-aware (UTC) + Field::new( + "timestamp_s_utc", + DataType::Timestamp(TimeUnit::Second, Some("UTC".into())), + false, + ), + Field::new( + "timestamp_ms_utc", + DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())), + false, + ), + Field::new( + "timestamp_us_utc", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Field::new( + "timestamp_ns_utc", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + false, + ), + // timestamp — all 4 units, tz-naive + Field::new("timestamp_s", DataType::Timestamp(TimeUnit::Second, None), false), + Field::new("timestamp_ms", DataType::Timestamp(TimeUnit::Millisecond, None), false), + Field::new("timestamp_us", DataType::Timestamp(TimeUnit::Microsecond, None), false), + Field::new("timestamp_ns", DataType::Timestamp(TimeUnit::Nanosecond, None), false), + // duration — all 4 units + Field::new("duration_s", DataType::Duration(TimeUnit::Second), false), + Field::new("duration_ms", DataType::Duration(TimeUnit::Millisecond), false), + Field::new("duration_us", DataType::Duration(TimeUnit::Microsecond), false), + Field::new("duration_ns", DataType::Duration(TimeUnit::Nanosecond), false), +``` + +- [ ] **Step 2: Add 12 corresponding arrays to the `RecordBatch::try_new` call** + +In the same `fn schema()`, find the line `Arc::new(Time64NanosecondArray::from(vec![3_600_000_000_000_i64])),` and insert immediately after it: + +```rust + // timestamp arrays (tz-aware UTC, all units; raw value = 1_000 ticks) + Arc::new( + TimestampSecondArray::from(vec![1_000_i64]).with_timezone("UTC"), + ), + Arc::new( + TimestampMillisecondArray::from(vec![1_000_i64]).with_timezone("UTC"), + ), + Arc::new( + TimestampMicrosecondArray::from(vec![1_000_i64]).with_timezone("UTC"), + ), + Arc::new( + TimestampNanosecondArray::from(vec![1_000_i64]).with_timezone("UTC"), + ), + // timestamp arrays (tz-naive, all units) + Arc::new(TimestampSecondArray::from(vec![1_000_i64])), + Arc::new(TimestampMillisecondArray::from(vec![1_000_i64])), + Arc::new(TimestampMicrosecondArray::from(vec![1_000_i64])), + Arc::new(TimestampNanosecondArray::from(vec![1_000_i64])), + // duration arrays (all units; 1_000 ticks each) + Arc::new(DurationSecondArray::from(vec![1_000_i64])), + Arc::new(DurationMillisecondArray::from(vec![1_000_i64])), + Arc::new(DurationMicrosecondArray::from(vec![1_000_i64])), + Arc::new(DurationNanosecondArray::from(vec![1_000_i64])), +``` + +- [ ] **Step 3: Run to confirm `schema()` test fails (panics from `todo!()`)** + +```bash +cargo test 'tests::schema' -- --nocapture 2>&1 | tail -20 +``` + +Expected: test panics with `not yet implemented`. + +--- + +### Task 4: Write failing raw buffer tests in `src/arrow_digester_core.rs` + +**Files:** +- Modify: `src/arrow_digester_core.rs` (internal `mod tests` — add after `digest_time64_nullable_bytes`) + +- [ ] **Step 1: Add timestamp array imports to the internal test module** + +In `src/arrow_digester_core.rs`, find the internal `mod tests` block (around line 1244). Add to the `use arrow::array::{ ... }` import: + +```rust +use arrow::array::{ + /* existing imports ... */ + TimestampMicrosecondArray, + DurationMicrosecondArray, +}; +``` + +The full updated import block (replace the existing one starting at `use arrow::{ array::{`): + +```rust + use arrow::{ + array::{ + ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array, Decimal128Array, + Decimal32Array, DurationMicrosecondArray, FixedSizeBinaryBuilder, Float16Array, + Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, + LargeBinaryArray, LargeListArray, LargeListBuilder, LargeStringArray, ListBuilder, + PrimitiveBuilder, RecordBatch, StringArray, StructArray, Time32SecondArray, + Time64MicrosecondArray, TimestampMicrosecondArray, UInt16Array, UInt32Array, + UInt64Array, UInt8Array, + }, + datatypes::Int32Type, + }; +``` + +- [ ] **Step 2: Add `digest_timestamp_nullable_bytes` test** + +Insert after `digest_time64_nullable_bytes` (around line 2156): + +```rust + // ── Timestamp / Duration ────────────────────────────────────────────── + + #[test] + fn digest_timestamp_nullable_bytes() { + // Microseconds since epoch: [0, None, 3_600_000_000] + let array = TimestampMicrosecondArray::from(vec![ + Some(0_i64), + None, + Some(3_600_000_000_i64), + ]) + .with_timezone("UTC"); + let mut digester = ArrowDigesterCore::::new( + &Schema::new(vec![Field::new( + "col", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + true, + )]), + false, + ); + digester.update( + &RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "col", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + true, + )])), + vec![Arc::new(array)], + ) + .unwrap(), + ); + + let buf = &digester.fields_digest_buffer["col"]; + let null_bit_vec = buf.null_bits.as_ref().expect("Expected nullable"); + let data_digest = buf.data.as_ref().expect("Expected data digest"); + + assert_eq!(null_bit_vec.len(), 3); + assert!(null_bit_vec[0], "index 0 should be valid"); + assert!(!null_bit_vec[1], "index 1 (None) should be null"); + assert!(null_bit_vec[2], "index 2 should be valid"); + + // Physical storage: int64 LE bytes for valid elements only + let mut manual = Sha256::new(); + manual.update(0_i64.to_le_bytes()); + manual.update(3_600_000_000_i64.to_le_bytes()); + assert_eq!(data_digest.clone().finalize(), manual.finalize()); + } + + #[test] + fn digest_duration_nullable_bytes() { + // Duration microseconds: [0, None, 3_600_000_000] + let array = DurationMicrosecondArray::from(vec![ + Some(0_i64), + None, + Some(3_600_000_000_i64), + ]); + let mut digester = ArrowDigesterCore::::new( + &Schema::new(vec![Field::new( + "col", + DataType::Duration(TimeUnit::Microsecond), + true, + )]), + false, + ); + digester.update( + &RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "col", + DataType::Duration(TimeUnit::Microsecond), + true, + )])), + vec![Arc::new(array)], + ) + .unwrap(), + ); + + let buf = &digester.fields_digest_buffer["col"]; + let null_bit_vec = buf.null_bits.as_ref().expect("Expected nullable"); + let data_digest = buf.data.as_ref().expect("Expected data digest"); + + assert_eq!(null_bit_vec.len(), 3); + assert!(null_bit_vec[0], "index 0 should be valid"); + assert!(!null_bit_vec[1], "index 1 (None) should be null"); + assert!(null_bit_vec[2], "index 2 should be valid"); + + let mut manual = Sha256::new(); + manual.update(0_i64.to_le_bytes()); + manual.update(3_600_000_000_i64.to_le_bytes()); + assert_eq!(data_digest.clone().finalize(), manual.finalize()); + } +``` + +- [ ] **Step 3: Run to confirm both raw buffer tests fail** + +```bash +cargo test digest_timestamp_nullable_bytes digest_duration_nullable_bytes -- --nocapture 2>&1 | tail -20 +``` + +Expected: both tests fail with panic from `todo!()`. + +--- + +### Task 5: Implement the Rust fix + +**Files:** +- Modify: `src/arrow_digester_core.rs` (lines ~941–944 in `array_digest_update`) + +- [ ] **Step 1: Replace the two `todo!()` arms** + +Find: + +```rust + DataType::Timestamp(_, _) => todo!(), + DataType::Time32(_) => Self::hash_fixed_size_array(effective_array, digest, 4), + DataType::Time64(_) => Self::hash_fixed_size_array(effective_array, digest, 8), + DataType::Duration(_) => todo!(), +``` + +Replace with: + +```rust + DataType::Timestamp(_, _) => { + // int64 physical storage; unit and tz are schema metadata, not data bytes + Self::hash_fixed_size_array(effective_array, digest, 8); + } + DataType::Time32(_) => Self::hash_fixed_size_array(effective_array, digest, 4), + DataType::Time64(_) => Self::hash_fixed_size_array(effective_array, digest, 8), + DataType::Duration(_) => { + // int64 physical storage; unit is schema metadata, not data bytes + Self::hash_fixed_size_array(effective_array, digest, 8); + } +``` + +- [ ] **Step 2: Verify it compiles** + +```bash +cargo build 2>&1 | tail -10 +``` + +Expected: `Finished` with no errors. + +--- + +### Task 6: Run all Rust tests and capture golden hash values + +- [ ] **Step 1: Run all tests and capture the printed hash values** + +```bash +cargo test timestamp_array_hashing duration_array_hashing -- --nocapture 2>&1 +``` + +Expected: both tests pass and print 12 lines like: + +``` +ts_s_utc: 000001... +ts_ms_utc: 000001... +ts_us_utc: 000001... +ts_ns_utc: 000001... +ts_s: 000001... +ts_ms: 000001... +ts_us: 000001... +ts_ns: 000001... +dur_s: 000001... +dur_ms: 000001... +dur_us: 000001... +dur_ns: 000001... +``` + +**Copy all 12 hex strings to a scratch file.** You will need them in Task 7 and in the Python golden parity tests (Task 10). + +- [ ] **Step 2: Run the full test suite to confirm all other tests still pass** + +```bash +cargo test 2>&1 | tail -20 +``` + +Expected: all tests pass, including the 5 new unit tests and 2 new raw buffer tests. + +--- + +### Task 7: Finalize unit test golden hash assertions + +**Files:** +- Modify: `tests/arrow_digester.rs` (replace `println!` + `starts_with` in `timestamp_array_hashing` and `duration_array_hashing`) + +- [ ] **Step 1: Replace `timestamp_array_hashing` with exact assertions** + +Replace the body of `timestamp_array_hashing` with (substituting the hex values captured in Task 6): + +```rust +#[test] +fn timestamp_array_hashing() { + let values = vec![Some(0_i64), None, Some(1_000_i64)]; + + // tz-aware UTC, all 4 units + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampSecondArray::from(values.clone()).with_timezone("UTC"), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampMillisecondArray::from(values.clone()).with_timezone("UTC"), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampMicrosecondArray::from(values.clone()).with_timezone("UTC"), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampNanosecondArray::from(values.clone()).with_timezone("UTC"), + )), + "" + ); + // tz-naive, all 4 units + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampSecondArray::from(values.clone()), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampMillisecondArray::from(values.clone()), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampMicrosecondArray::from(values.clone()), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &TimestampNanosecondArray::from(values.clone()), + )), + "" + ); +} +``` + +- [ ] **Step 2: Replace `duration_array_hashing` with exact assertions** + +```rust +#[test] +fn duration_array_hashing() { + let values = vec![Some(0_i64), None, Some(1_000_i64)]; + + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &DurationSecondArray::from(values.clone()), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &DurationMillisecondArray::from(values.clone()), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &DurationMicrosecondArray::from(values.clone()), + )), + "" + ); + assert_eq!( + hex::encode(ArrowDigester::hash_array( + &DurationNanosecondArray::from(values.clone()), + )), + "" + ); +} +``` + +- [ ] **Step 3: Run to confirm exact assertions pass** + +```bash +cargo test timestamp_array_hashing duration_array_hashing 2>&1 | tail -10 +``` + +Expected: both pass. + +--- + +### Task 8: Update the `schema()` golden hash + +**Files:** +- Modify: `tests/arrow_digester.rs` (the existing `assert_eq!` for `hash_record_batch` in `fn schema()`) + +- [ ] **Step 1: Run the schema test and capture the new hash** + +```bash +cargo test 'tests::schema' -- --nocapture 2>&1 | tail -10 +``` + +Expected: test fails because the golden hash `"000001487059003be1..."` no longer matches (12 new columns added). The failure message will show the actual new hash value. + +- [ ] **Step 2: Update the `hash_record_batch` assertion with the new hash** + +In `fn schema()`, find: + +```rust + assert_eq!( + encode(ArrowDigester::hash_record_batch( + &batch, + HasherConfig::default() + )), + "000001487059003be1a84dbe29ba6e90ea50798a76d22e46e221b6a0c332421dc4062e" + ); +``` + +Replace the hash string with the value printed in Step 1. + +- [ ] **Step 3: Run `schema()` to confirm it passes** + +```bash +cargo test 'tests::schema' 2>&1 | tail -10 +``` + +Expected: PASS. + +--- + +### Task 9: `cargo fmt`, full test run, and commit + +- [ ] **Step 1: Format** + +```bash +cargo fmt +``` + +- [ ] **Step 2: Run the full test suite one final time** + +```bash +cargo test 2>&1 | tail -10 +``` + +Expected: all tests pass. + +- [ ] **Step 3: Commit** + +```bash +cd starfix +git add src/arrow_digester_core.rs tests/arrow_digester.rs +git commit -m "feat: add timestamp and duration support in leaf data hashing (ITL-438) + +- Replace todo!() for DataType::Timestamp and DataType::Duration with + hash_fixed_size_array(..., 8) — int64 physical storage +- Add timestamp_array_hashing, duration_array_hashing, timestamp_units_differ, + timestamp_tz_differs, duration_units_differ unit tests +- Add digest_timestamp_nullable_bytes, digest_duration_nullable_bytes raw buffer tests +- Update comprehensive schema() golden hash for new columns + +Co-Authored-By: Claude Sonnet 4.6 " +``` + +--- + +## Phase 2: Python (`nauticalab/starfix-python`) + +### Task 10: Write failing Python schema serialization tests + +**Files:** +- Modify: `tests/test_arrow_digester.py` (add to `TestSchemaSerialization` class) + +- [ ] **Step 1: Add `test_timestamp_types_in_schema`** + +Append to the `TestSchemaSerialization` class: + +```python + def test_timestamp_types_in_schema(self): + schema = pa.schema([ + pa.field("ts_utc_s", pa.timestamp("s", tz="UTC"), nullable=False), + pa.field("ts_utc_ms", pa.timestamp("ms", tz="UTC"), nullable=False), + pa.field("ts_utc_us", pa.timestamp("us", tz="UTC"), nullable=False), + pa.field("ts_utc_ns", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("ts_naive_us", pa.timestamp("us"), nullable=False), + ]) + s = _serialized_schema(schema) + assert '{"Timestamp":["Second","UTC"]}' in s + assert '{"Timestamp":["Millisecond","UTC"]}' in s + assert '{"Timestamp":["Microsecond","UTC"]}' in s + assert '{"Timestamp":["Nanosecond","UTC"]}' in s + assert '{"Timestamp":["Microsecond",null]}' in s + + def test_duration_types_in_schema(self): + schema = pa.schema([ + pa.field("dur_s", pa.duration("s"), nullable=False), + pa.field("dur_ms", pa.duration("ms"), nullable=False), + pa.field("dur_us", pa.duration("us"), nullable=False), + pa.field("dur_ns", pa.duration("ns"), nullable=False), + ]) + s = _serialized_schema(schema) + assert '{"Duration":"Second"}' in s + assert '{"Duration":"Millisecond"}' in s + assert '{"Duration":"Microsecond"}' in s + assert '{"Duration":"Nanosecond"}' in s +``` + +- [ ] **Step 2: Run to confirm the new schema tests pass** + +The schema serialization tests should NOT fail — `_data_type_to_value` already handles timestamp/duration correctly. Confirm: + +```bash +cd starfix-python +pytest tests/test_arrow_digester.py::TestSchemaSerialization -v 2>&1 | tail -20 +``` + +Expected: all pass (including the two new schema tests). + +--- + +### Task 11: Write failing Python golden parity tests + +**Files:** +- Modify: `tests/test_golden_parity.py` (add a new `TestTimestampDurationHashing` class) + +- [ ] **Step 1: Add the `TestTimestampDurationHashing` class** + +Append to `tests/test_golden_parity.py` (use the hash values captured from Task 6 in Phase 1): + +```python +# --------------------------------------------------------------------------- +# Timestamp and Duration hashing — golden values match Rust (ITL-438) +# --------------------------------------------------------------------------- + + +class TestTimestampDurationHashing: + """Golden-hash parity for timestamp and duration types. + + All expected values were generated by the Rust starfix test suite + (timestamp_array_hashing / duration_array_hashing in tests/arrow_digester.rs). + Array contents: [0, None, 1_000] as raw i64 values. + """ + + # -- Timestamp (tz-aware UTC) ------------------------------------------ + + def test_timestamp_second_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("s", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_timestamp_millisecond_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ms", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_timestamp_microsecond_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("us", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_timestamp_nanosecond_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ns", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "" + + # -- Timestamp (tz-naive) ---------------------------------------------- + + def test_timestamp_second_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("s")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_timestamp_millisecond_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ms")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_timestamp_microsecond_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("us")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_timestamp_nanosecond_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ns")) + assert ArrowDigester.hash_array(arr).hex() == "" + + # -- Timestamp: unit and tz produce different hashes ------------------- + + def test_timestamp_units_differ(self): + values = [1_000, 2_000] + hashes = [ + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("s"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("ms"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("us"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("ns"))).hex(), + ] + assert len(set(hashes)) == 4, "all 4 timestamp units must produce distinct hashes" + + def test_timestamp_tz_differs(self): + values = [1_000, 2_000] + naive = ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("us"))).hex() + utc = ArrowDigester.hash_array( + pa.array(values, type=pa.timestamp("us", tz="UTC")) + ).hex() + assert naive != utc, "tz-naive and tz=UTC must produce different hashes" + + # -- Duration ---------------------------------------------------------- + + def test_duration_second(self): + arr = pa.array([0, None, 1_000], type=pa.duration("s")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_duration_millisecond(self): + arr = pa.array([0, None, 1_000], type=pa.duration("ms")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_duration_microsecond(self): + arr = pa.array([0, None, 1_000], type=pa.duration("us")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_duration_nanosecond(self): + arr = pa.array([0, None, 1_000], type=pa.duration("ns")) + assert ArrowDigester.hash_array(arr).hex() == "" + + def test_duration_units_differ(self): + values = [1_000, 2_000] + hashes = [ + ArrowDigester.hash_array(pa.array(values, type=pa.duration("s"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.duration("ms"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.duration("us"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.duration("ns"))).hex(), + ] + assert len(set(hashes)) == 4, "all 4 duration units must produce distinct hashes" +``` + +- [ ] **Step 2: Run to confirm all golden parity tests fail** + +```bash +pytest tests/test_golden_parity.py::TestTimestampDurationHashing -v 2>&1 | tail -30 +``` + +Expected: all 15 tests fail with `NotImplementedError: Unsupported leaf type: ...`. + +--- + +### Task 12: Implement the Python fix + +**Files:** +- Modify: `src/starfix/arrow_digester.py` (`_element_size_for_type` function, around line 629) + +- [ ] **Step 1: Add `is_timestamp` and `is_duration` checks** + +Find in `_element_size_for_type`: + +```python + if pa.types.is_time32(dt): + return 4 + if pa.types.is_time64(dt): + return 8 + if pa.types.is_decimal(dt): +``` + +Replace with: + +```python + if pa.types.is_time32(dt): + return 4 + if pa.types.is_time64(dt): + return 8 + if pa.types.is_timestamp(dt): + return 8 # int64 physical storage; unit/tz are schema metadata + if pa.types.is_duration(dt): + return 8 # int64 physical storage; unit is schema metadata + if pa.types.is_decimal(dt): +``` + +- [ ] **Step 2: Run the golden parity tests to confirm they now pass** + +```bash +pytest tests/test_golden_parity.py::TestTimestampDurationHashing -v 2>&1 | tail -30 +``` + +Expected: all 15 tests pass. + +- [ ] **Step 3: Run the full Python test suite** + +```bash +pytest tests/ -v 2>&1 | tail -20 +``` + +Expected: all tests pass. + +- [ ] **Step 4: Commit** + +```bash +cd starfix-python +git add src/starfix/arrow_digester.py tests/test_arrow_digester.py tests/test_golden_parity.py +git commit -m "feat: add timestamp and duration support in leaf data hashing (ITL-438) + +- Add is_timestamp and is_duration checks to _element_size_for_type (8 bytes each) +- Add test_timestamp_types_in_schema and test_duration_types_in_schema +- Add TestTimestampDurationHashing golden parity tests (15 cases) + matching Rust starfix byte-for-byte + +Co-Authored-By: Claude Sonnet 4.6 " +``` + +--- + +## Phase 3: Pull Requests + +### Task 13: Create the starfix (Rust) PR + +- [ ] **Step 1: Push the branch** + +```bash +cd starfix +git push -u origin eywalker/itl-438-add-support-for-timestamp-and-duration-types-in-leaf-data +``` + +- [ ] **Step 2: Create the PR targeting `main`** + +```bash +gh pr create \ + --title "feat: add timestamp and duration support in leaf data hashing (ITL-438)" \ + --base main \ + --body "$(cat <<'EOF' +## Summary + +- Replaces `todo!()` for `DataType::Timestamp(_, _)` and `DataType::Duration(_)` in `array_digest_update` with `hash_fixed_size_array(..., 8)` — both types are physically stored as int64 (8 bytes), identical to `time64` +- Adds unit tests: `timestamp_array_hashing`, `duration_array_hashing`, `timestamp_units_differ`, `timestamp_tz_differs`, `duration_units_differ` +- Adds raw buffer tests: `digest_timestamp_nullable_bytes`, `digest_duration_nullable_bytes` +- Updates the comprehensive `schema()` golden hash for the 12 new columns + +## Polars note + +`timestamp[s]` and `duration[s]` are silently coerced to `ms` by Polars — documented in `docs/metamorphic/specs/2026-06-28-timestamp-duration-hashing-design.md`. starfix does not normalise this; the hash reflects the actual Arrow type. + +## Test plan + +- [ ] `cargo test` passes locally +- [ ] `cargo fmt` applied + +Closes ITL-438 +🤖 Generated with [Claude Code](https://claude.com/claude-code) +EOF +)" +``` + +--- + +### Task 14: Create the starfix-python PR + +- [ ] **Step 1: Push the branch** + +```bash +cd starfix-python +git push -u origin eywalker/itl-438-add-support-for-timestamp-and-duration-types-in-leaf-data +``` + +- [ ] **Step 2: Create the PR targeting `main`** + +```bash +gh pr create \ + --title "feat: add timestamp and duration support in leaf data hashing (ITL-438)" \ + --base main \ + --body "$(cat <<'EOF' +## Summary + +- Adds `is_timestamp` and `is_duration` checks to `_element_size_for_type` in `arrow_digester.py` (return 8 — int64 physical storage) +- Adds `test_timestamp_types_in_schema` and `test_duration_types_in_schema` to `test_arrow_digester.py` +- Adds `TestTimestampDurationHashing` class in `test_golden_parity.py` with 15 tests, golden hash values taken from the Rust `starfix` implementation + +## Dependency + +Golden hash values were generated from `nauticalab/starfix` after implementing the Rust fix. Python and Rust hashes are byte-for-byte identical. + +## Test plan + +- [ ] `pytest tests/` passes locally + +Closes ITL-438 +🤖 Generated with [Claude Code](https://claude.com/claude-code) +EOF +)" +``` From 6589652930d6bfbeaa4591dc58b98b43664dde21 Mon Sep 17 00:00:00 2001 From: "agent-kurodo[bot]" <268466204+agent-kurodo[bot]@users.noreply.github.com> Date: Sun, 28 Jun 2026 23:04:31 +0000 Subject: [PATCH 5/5] feat: add timestamp and duration support in leaf data hashing (ITL-438) - Add is_timestamp and is_duration checks to _element_size_for_type (8 bytes each) - Add test_timestamp_types_in_schema and test_duration_types_in_schema - Add TestTimestampDurationHashing golden parity tests (15 cases) matching Rust starfix byte-for-byte Co-Authored-By: Claude Sonnet 4.6 --- src/starfix/arrow_digester.py | 4 ++ tests/test_arrow_digester.py | 28 ++++++++++ tests/test_golden_parity.py | 98 +++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+) diff --git a/src/starfix/arrow_digester.py b/src/starfix/arrow_digester.py index cfca8ff..5110c17 100644 --- a/src/starfix/arrow_digester.py +++ b/src/starfix/arrow_digester.py @@ -642,6 +642,10 @@ def _element_size_for_type(dt: pa.DataType) -> int | None: return 4 if pa.types.is_time64(dt): return 8 + if pa.types.is_timestamp(dt): + return 8 # int64 physical storage; unit/tz are schema metadata + if pa.types.is_duration(dt): + return 8 # int64 physical storage; unit is schema metadata if pa.types.is_decimal(dt): return dt.bit_width // 8 if pa.types.is_fixed_size_binary(dt): diff --git a/tests/test_arrow_digester.py b/tests/test_arrow_digester.py index 9d9e083..0d5f2d6 100644 --- a/tests/test_arrow_digester.py +++ b/tests/test_arrow_digester.py @@ -67,6 +67,34 @@ def test_struct_fields_sorted_in_schema(self): # a_field should appear before z_field in the Struct array assert s.index('"a_field"') < s.index('"z_field"') + def test_timestamp_types_in_schema(self): + schema = pa.schema([ + pa.field("ts_utc_s", pa.timestamp("s", tz="UTC"), nullable=False), + pa.field("ts_utc_ms", pa.timestamp("ms", tz="UTC"), nullable=False), + pa.field("ts_utc_us", pa.timestamp("us", tz="UTC"), nullable=False), + pa.field("ts_utc_ns", pa.timestamp("ns", tz="UTC"), nullable=False), + pa.field("ts_naive_us", pa.timestamp("us"), nullable=False), + ]) + s = _serialized_schema(schema) + assert '{"Timestamp":["Second","UTC"]}' in s + assert '{"Timestamp":["Millisecond","UTC"]}' in s + assert '{"Timestamp":["Microsecond","UTC"]}' in s + assert '{"Timestamp":["Nanosecond","UTC"]}' in s + assert '{"Timestamp":["Microsecond",null]}' in s + + def test_duration_types_in_schema(self): + schema = pa.schema([ + pa.field("dur_s", pa.duration("s"), nullable=False), + pa.field("dur_ms", pa.duration("ms"), nullable=False), + pa.field("dur_us", pa.duration("us"), nullable=False), + pa.field("dur_ns", pa.duration("ns"), nullable=False), + ]) + s = _serialized_schema(schema) + assert '{"Duration":"Second"}' in s + assert '{"Duration":"Millisecond"}' in s + assert '{"Duration":"Microsecond"}' in s + assert '{"Duration":"Nanosecond"}' in s + # ── Schema hashing (golden values from Rust) ────────────────────────── diff --git a/tests/test_golden_parity.py b/tests/test_golden_parity.py index 60924c9..df28334 100644 --- a/tests/test_golden_parity.py +++ b/tests/test_golden_parity.py @@ -607,3 +607,101 @@ def test_batch_split_independence(self): hash_split = digester.finalize() assert hash_combined == hash_split + + +# --------------------------------------------------------------------------- +# Timestamp and Duration hashing — golden values match Rust (ITL-438) +# --------------------------------------------------------------------------- + + +class TestTimestampDurationHashing: + """Golden-hash parity for timestamp and duration types. + + All expected values were generated by the Rust starfix test suite + (timestamp_array_hashing / duration_array_hashing in tests/arrow_digester.rs). + Array contents: [0, None, 1_000] as raw i64 values. + """ + + # -- Timestamp (tz-aware UTC) ------------------------------------------ + + def test_timestamp_second_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("s", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "0000017cef6166a14741f94906bd3146ffa197d1b5756d594eb46304b678b21e44c40b" + + def test_timestamp_millisecond_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ms", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "000001fd253a835218d5605ce8bb348699f973b93e29fbe21fe6e532468f988ece4ce5" + + def test_timestamp_microsecond_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("us", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "0000013af741ca0fe5bd48d0bd39e1993c60e7c032f23c4cf4be6c11cc4b50ce56013e" + + def test_timestamp_nanosecond_utc(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ns", tz="UTC")) + assert ArrowDigester.hash_array(arr).hex() == "000001f5b86904b4b7f2c6a18c7c4febda142bf622e4d01752f42b8f832c0a68a4d9c0" + + # -- Timestamp (tz-naive) ---------------------------------------------- + + def test_timestamp_second_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("s")) + assert ArrowDigester.hash_array(arr).hex() == "0000016ee5249d352c03cc820af39e98b7043993d467207cb6ca34c8d13cfacf141e30" + + def test_timestamp_millisecond_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ms")) + assert ArrowDigester.hash_array(arr).hex() == "000001ef46619d2be8259335ed0943f42ea3ced19ff6d75471a64b2f03aea89afa4842" + + def test_timestamp_microsecond_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("us")) + assert ArrowDigester.hash_array(arr).hex() == "0000017e624eb2847f79f940d7123980b076aea1c1ecd3adaa34bdae24fa00f05afda3" + + def test_timestamp_nanosecond_naive(self): + arr = pa.array([0, None, 1_000], type=pa.timestamp("ns")) + assert ArrowDigester.hash_array(arr).hex() == "000001f677ef0e42e9e40ea092471f82283212833ab6b4ff9bb45e86e88590f8d15796" + + # -- Timestamp: unit and tz produce different hashes ------------------- + + def test_timestamp_units_differ(self): + values = [1_000, 2_000] + hashes = [ + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("s"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("ms"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("us"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("ns"))).hex(), + ] + assert len(set(hashes)) == 4, "all 4 timestamp units must produce distinct hashes" + + def test_timestamp_tz_differs(self): + values = [1_000, 2_000] + naive = ArrowDigester.hash_array(pa.array(values, type=pa.timestamp("us"))).hex() + utc = ArrowDigester.hash_array( + pa.array(values, type=pa.timestamp("us", tz="UTC")) + ).hex() + assert naive != utc, "tz-naive and tz=UTC must produce different hashes" + + # -- Duration ---------------------------------------------------------- + + def test_duration_second(self): + arr = pa.array([0, None, 1_000], type=pa.duration("s")) + assert ArrowDigester.hash_array(arr).hex() == "0000013540d23a4abf1dfbc939a9e5514be69e364cec466ef900fa35050dc3cf2994fa" + + def test_duration_millisecond(self): + arr = pa.array([0, None, 1_000], type=pa.duration("ms")) + assert ArrowDigester.hash_array(arr).hex() == "000001564a87e1e07898af07a4a212e3db83f911ac287134f28c772bf5fb4da683402c" + + def test_duration_microsecond(self): + arr = pa.array([0, None, 1_000], type=pa.duration("us")) + assert ArrowDigester.hash_array(arr).hex() == "0000017904faf3043cf870f0c139cba0282cdaf11c590aeb80f2a0d6b103893ba5da2a" + + def test_duration_nanosecond(self): + arr = pa.array([0, None, 1_000], type=pa.duration("ns")) + assert ArrowDigester.hash_array(arr).hex() == "00000189a38c5a7adf4b19b3e0d467f042ab2389fed8ae5f1d2af8555dd6131478a49b" + + def test_duration_units_differ(self): + values = [1_000, 2_000] + hashes = [ + ArrowDigester.hash_array(pa.array(values, type=pa.duration("s"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.duration("ms"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.duration("us"))).hex(), + ArrowDigester.hash_array(pa.array(values, type=pa.duration("ns"))).hex(), + ] + assert len(set(hashes)) == 4, "all 4 duration units must produce distinct hashes"