From 575510d688b7a8a65a49d84cc417d1932cae83a8 Mon Sep 17 00:00:00 2001 From: Leonid Ryzhyk Date: Tue, 14 Jul 2026 21:43:51 -0700 Subject: [PATCH 1/7] Accept +00:00 and other zero-offset timezones as UTC Arrow encodes every timezone-aware timestamp as a UTC instant; the timezone string is only a display hint. Previously, timestamp deserializer only recognized the "UTC" string as UTC timezone specifier. This commit adds support for other zero-offset designators (+00:00, -00:00, +0000, -0000, Z) alongside "UTC" through a shared is_utc_timezone helper used by both the deserialization and serialization paths. --- serde_arrow/src/internal/chrono.rs | 10 +++ .../deserialization/timestamp_deserializer.rs | 5 +- .../serialization/timestamp_builder.rs | 3 +- .../test_with_arrow/impls/arrow_timestamp.rs | 66 +++++++++++++++++++ 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/serde_arrow/src/internal/chrono.rs b/serde_arrow/src/internal/chrono.rs index f2ce2c4c..9c2c1a98 100644 --- a/serde_arrow/src/internal/chrono.rs +++ b/serde_arrow/src/internal/chrono.rs @@ -29,6 +29,16 @@ pub fn matches_naive_time(s: &str) -> bool { parsing::match_naive_time(s).matches() } +/// Check whether an Arrow timezone string denotes UTC (a zero offset). +/// +/// Arrow encodes every timezone-aware timestamp as a UTC instant; the timezone +/// string only records how to display it. Recognize the name `UTC` and the fixed +/// zero-offset designators. +pub fn is_utc_timezone(tz: &str) -> bool { + tz.eq_ignore_ascii_case("utc") + || matches!(tz, "Z" | "z" | "+00:00" | "-00:00" | "+0000" | "-0000") +} + /// Parse `s` as a span pub fn parse_span(s: &str) -> Result> { parsing::match_span(s).into_result("Span") diff --git a/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs b/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs index ea188f4d..e61e8bfa 100644 --- a/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs +++ b/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs @@ -6,6 +6,7 @@ use marrow::{ use serde::de::Visitor; use crate::internal::{ + chrono::is_utc_timezone, error::{fail, set_default, try_, Context, ContextSupport, Result}, utils::array_view_ext::ViewAccess, }; @@ -75,9 +76,9 @@ impl<'a> TimestampDeserializer<'a> { fn is_utc_timestamp(timezone: Option<&str>) -> Result { match timezone { - Some(tz) if tz.to_lowercase() == "utc" => Ok(true), - Some(tz) => fail!("unsupported timezone: {} is not supported", tz), None => Ok(false), + Some(tz) if is_utc_timezone(tz) => Ok(true), + Some(tz) => fail!("unsupported timezone: {} is not supported", tz), } } diff --git a/serde_arrow/src/internal/serialization/timestamp_builder.rs b/serde_arrow/src/internal/serialization/timestamp_builder.rs index 2f4670b2..1d8c2bb3 100644 --- a/serde_arrow/src/internal/serialization/timestamp_builder.rs +++ b/serde_arrow/src/internal/serialization/timestamp_builder.rs @@ -7,6 +7,7 @@ use marrow::{ use serde::Serialize; use crate::internal::{ + chrono::is_utc_timezone, error::{fail, set_default, Context, ContextSupport, Result}, serialization::utils::impl_serializer, utils::array_ext::{ArrayExt, ScalarArrayExt}, @@ -88,7 +89,7 @@ impl TimestampBuilder { fn is_utc_tz(tz: Option<&str>) -> Result { match tz { None => Ok(false), - Some(tz) if tz.to_uppercase() == "UTC" => Ok(true), + Some(tz) if is_utc_timezone(tz) => Ok(true), Some(tz) => fail!("timezone {tz} is not supported"), } } diff --git a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs index 2e31adc2..43338512 100644 --- a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs +++ b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs @@ -269,3 +269,69 @@ mod schema_tracing { .check_nulls(&[&[false, false]]); } } + +/// A timezone-aware Arrow timestamp is a UTC instant, so every zero-offset +/// timezone designator must deserialize. Iceberg tags its `timestamptz` columns +/// with the fixed offset `+00:00`. +mod timezone_utc_designators { + use super::*; + + const UTC_DESIGNATORS: &[&str] = + &["+00:00", "-00:00", "+0000", "-0000", "Z", "z", "UTC", "utc"]; + + #[test] + fn deserializes_every_utc_designator_as_zulu() { + // 2025-01-20T19:30:42 UTC as microseconds since the epoch. + let micros = NaiveDateTime::parse_from_str("2025-01-20T19:30:42", "%Y-%m-%dT%H:%M:%S") + .unwrap() + .and_utc() + .timestamp_micros(); + + for tz in UTC_DESIGNATORS { + let array = Array::Timestamp(TimestampArray { + unit: TimeUnit::Microsecond, + timezone: Some(String::from(*tz)), + validity: None, + values: vec![micros], + }); + let view = array.as_view(); + let deserializer = Deserializer::from_marrow(&[tz_field(tz)], &[view]) + .unwrap_or_else(|e| panic!("timezone {tz:?} rejected: {e}")); + let actual = Vec::>::deserialize(deserializer).unwrap(); + + assert_eq!( + actual, + [Item(String::from("2025-01-20T19:30:42Z"))], + "timezone {tz:?}" + ); + } + } + + #[test] + fn rejects_non_utc_offset() { + let array = Array::Timestamp(TimestampArray { + unit: TimeUnit::Microsecond, + timezone: Some(String::from("+01:00")), + validity: None, + values: vec![0], + }); + let view = array.as_view(); + let err = match Deserializer::from_marrow(&[tz_field("+01:00")], &[view]) { + Ok(_) => panic!("expected the +01:00 offset to be rejected"), + Err(err) => err, + }; + assert!( + err.to_string().contains("+01:00"), + "unexpected error: {err}" + ); + } + + fn tz_field(tz: &str) -> Field { + Field { + name: String::from("item"), + data_type: DataType::Timestamp(TimeUnit::Microsecond, Some(String::from(tz))), + nullable: false, + metadata: Default::default(), + } + } +} From 4e6587e12dc146112f4971210ef80afeae77d59f Mon Sep 17 00:00:00 2001 From: Leonid Ryzhyk Date: Sat, 18 Jul 2026 13:51:38 -0700 Subject: [PATCH 2/7] Address review feedback. - Update stale docs. - Test the serialization path. - Update changelog. Signed-off-by: Leonid Ryzhyk --- Changes.md | 5 +++ serde_arrow/src/internal/schema/mod.rs | 3 +- .../test_with_arrow/impls/arrow_timestamp.rs | 38 +++++++++++++++---- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/Changes.md b/Changes.md index 94da8b24..480d5ace 100644 --- a/Changes.md +++ b/Changes.md @@ -1,5 +1,10 @@ # Change log +## Development + +- Accept zero-offset timezone designators (`+00:00`, `-00:00`, `+0000`, `-0000`, + `Z`, `z`) alongside `Utc` as UTC when serializing and deserializing timestamps + ## 0.15.0-rc.1 Breaking changes: diff --git a/serde_arrow/src/internal/schema/mod.rs b/serde_arrow/src/internal/schema/mod.rs index 2581d413..d6adca55 100644 --- a/serde_arrow/src/internal/schema/mod.rs +++ b/serde_arrow/src/internal/schema/mod.rs @@ -108,7 +108,8 @@ pub trait SchemaLike: Sized + Sealed { /// - date objects: `"Date32"`, `"Date64"` /// - datetime objects: `"Timestamp(unit, optional_timezone)"` with `unit` being one of /// `Second`, `Millisecond`, `Microsecond`, `Nanosecond` and `optional_timezone` being either - /// `None` or `Some("Utc")`. + /// `None` or a UTC zero-offset designator (`Some("Utc")`, `Some("Z")`, `Some("+00:00")`, + /// `Some("-00:00")`, `Some("+0000")`, or `Some("-0000")`). /// - time objects: `"Time32(unit)"`, `"Time64(unit)"` with unit being one of `Second`, /// `Millisecond`, `Microsecond`, `Nanosecond`. /// - durations: `"Duration(unit)"` with unit being one of `Second`, `Millisecond`, diff --git a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs index 43338512..f2cfe771 100644 --- a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs +++ b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs @@ -281,18 +281,12 @@ mod timezone_utc_designators { #[test] fn deserializes_every_utc_designator_as_zulu() { - // 2025-01-20T19:30:42 UTC as microseconds since the epoch. - let micros = NaiveDateTime::parse_from_str("2025-01-20T19:30:42", "%Y-%m-%dT%H:%M:%S") - .unwrap() - .and_utc() - .timestamp_micros(); - for tz in UTC_DESIGNATORS { let array = Array::Timestamp(TimestampArray { unit: TimeUnit::Microsecond, timezone: Some(String::from(*tz)), validity: None, - values: vec![micros], + values: vec![zulu_micros()], }); let view = array.as_view(); let deserializer = Deserializer::from_marrow(&[tz_field(tz)], &[view]) @@ -307,6 +301,28 @@ mod timezone_utc_designators { } } + /// Tagging the schema with a zero-offset designator sets the builder to UTC, + /// so a `Z`-suffixed string parses instead of being rejected as non-naive. + #[test] + fn serializes_every_utc_designator_from_zulu_string() { + for tz in UTC_DESIGNATORS { + let mut builder = ArrayBuilder::from_marrow(&[tz_field(tz)]) + .unwrap_or_else(|e| panic!("timezone {tz:?} rejected: {e}")); + [Item(String::from("2025-01-20T19:30:42Z"))] + .serialize(Serializer::new(&mut builder)) + .unwrap_or_else(|e| panic!("timezone {tz:?}: {e}")); + + let arrays = builder.to_marrow().unwrap(); + let [array] = <[_; 1]>::try_from(arrays).unwrap(); + let Array::Timestamp(array) = array else { + panic!("timezone {tz:?}: expected a timestamp array"); + }; + + assert_eq!(array.values, [zulu_micros()], "timezone {tz:?}"); + assert_eq!(array.timezone.as_deref(), Some(*tz), "timezone {tz:?}"); + } + } + #[test] fn rejects_non_utc_offset() { let array = Array::Timestamp(TimestampArray { @@ -326,6 +342,14 @@ mod timezone_utc_designators { ); } + /// `2025-01-20T19:30:42Z` as microseconds since the epoch. + fn zulu_micros() -> i64 { + NaiveDateTime::parse_from_str("2025-01-20T19:30:42", "%Y-%m-%dT%H:%M:%S") + .unwrap() + .and_utc() + .timestamp_micros() + } + fn tz_field(tz: &str) -> Field { Field { name: String::from("item"), From 2dd835933df14b5029fd3f7c5e903cf795f573c0 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 19 Jul 2026 10:39:05 +0200 Subject: [PATCH 3/7] Add shoutout to changelog --- Changes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Changes.md b/Changes.md index 480d5ace..e73beae4 100644 --- a/Changes.md +++ b/Changes.md @@ -5,6 +5,11 @@ - Accept zero-offset timezone designators (`+00:00`, `-00:00`, `+0000`, `-0000`, `Z`, `z`) alongside `Utc` as UTC when serializing and deserializing timestamps +### Thanks + +- [@ryzhyk](https://github.com/ryzhyk) added support to for zero offset timezone + designators ([#320](https://github.com/chmp/serde_arrow/pull/320)) + ## 0.15.0-rc.1 Breaking changes: From 64211619fab51979a4d9bbdab9e0e9dc95f42986 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 19 Jul 2026 10:49:37 +0200 Subject: [PATCH 4/7] Add a test that the offsets are also recognized when building the arrays --- .../test_with_arrow/impls/arrow_timestamp.rs | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs index f2cfe771..95cf9693 100644 --- a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs +++ b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs @@ -323,6 +323,31 @@ mod timezone_utc_designators { } } + #[test] + fn serializes_zero_offset_designators_from_string_values() { + let items = [ + Item(String::from("2025-01-20T19:30:42+0000")), + Item(String::from("2025-01-20T19:30:42+00:00")), + Item(String::from("2025-01-20T19:30:42-0000")), + Item(String::from("2025-01-20T19:30:42-00:00")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42z")), + ]; + let mut builder = ArrayBuilder::from_marrow(&[tz_field("UTC")]).unwrap(); + items + .serialize(Serializer::new(&mut builder)) + .unwrap_or_else(|e| panic!("{e}")); + + let arrays = builder.to_marrow().unwrap(); + let [array] = <[_; 1]>::try_from(arrays).unwrap(); + let Array::Timestamp(array) = array else { + panic!("expected a timestamp array"); + }; + + assert_eq!(array.values, vec![zulu_micros(); items.len()]); + assert_eq!(array.timezone.as_deref(), Some("UTC")); + } + #[test] fn rejects_non_utc_offset() { let array = Array::Timestamp(TimestampArray { From a8dfb374ddc3753c6c59437afa5510f7a6252fea Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 19 Jul 2026 10:57:42 +0200 Subject: [PATCH 5/7] Support zero offsets in schema tracing --- serde_arrow/src/internal/chrono.rs | 2 +- .../test_with_arrow/impls/arrow_timestamp.rs | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/serde_arrow/src/internal/chrono.rs b/serde_arrow/src/internal/chrono.rs index 9c2c1a98..44e1469f 100644 --- a/serde_arrow/src/internal/chrono.rs +++ b/serde_arrow/src/internal/chrono.rs @@ -355,7 +355,7 @@ mod parsing { /// /// Note: this function is more permissive than some libraries (e.g., jiff) pub fn match_utc_timezone(s: &str) -> Result<(&str, &str), &str> { - for prefix in ["Z", "+0000", "+00:00"] { + for prefix in ["Z", "z", "+0000", "-0000", "+00:00", "-00:00"] { if let Some((prefix, rest)) = split_prefix(s, prefix) { return Ok((rest, prefix)); } diff --git a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs index 95cf9693..efde28cb 100644 --- a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs +++ b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs @@ -184,6 +184,36 @@ mod schema_tracing { .check_nulls(&[&[false, true, false]]); } + #[test] + fn utc_zero_offset_designators_as_timestamp_tracing() { + let items = [ + Item(String::from("2025-01-20T19:30:42+0000")), + Item(String::from("2025-01-20T19:30:42+00:00")), + Item(String::from("2025-01-20T19:30:42-0000")), + Item(String::from("2025-01-20T19:30:42-00:00")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42z")), + ]; + let expected = [ + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + ]; + + Test::new() + .with_schema(json!([{ + "name": "item", + "data_type": "Timestamp(Millisecond, Some(\"UTC\"))", + }])) + .trace_schema_from_samples(&items, TracingOptions::default().guess_dates(true)) + .serialize(&items) + .deserialize(&expected) + .check_nulls(&[&[false, false, false, false, false, false]]); + } + #[test] fn utc_tracing_string_only_with_invalid() { let items = [ From 760d30ed926234600b87aa322dd6f8df5609f723 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 19 Jul 2026 10:57:54 +0200 Subject: [PATCH 6/7] Update changelog --- Changes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index e73beae4..b20a51d0 100644 --- a/Changes.md +++ b/Changes.md @@ -4,10 +4,12 @@ - Accept zero-offset timezone designators (`+00:00`, `-00:00`, `+0000`, `-0000`, `Z`, `z`) alongside `Utc` as UTC when serializing and deserializing timestamps +- Infer strings with zero-offset timezone designators as UTC timestamps when + date guessing is enabled ### Thanks -- [@ryzhyk](https://github.com/ryzhyk) added support to for zero offset timezone +- [@ryzhyk](https://github.com/ryzhyk) added support for zero-offset timezone designators ([#320](https://github.com/chmp/serde_arrow/pull/320)) ## 0.15.0-rc.1 From 56eb0e7bec9c84de025d253d5f402ecd1fe525c1 Mon Sep 17 00:00:00 2001 From: Christopher Prohm Date: Sun, 19 Jul 2026 11:08:08 +0200 Subject: [PATCH 7/7] Tune changelog --- Changes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index b20a51d0..029a14af 100644 --- a/Changes.md +++ b/Changes.md @@ -3,7 +3,7 @@ ## Development - Accept zero-offset timezone designators (`+00:00`, `-00:00`, `+0000`, `-0000`, - `Z`, `z`) alongside `Utc` as UTC when serializing and deserializing timestamps + `Z`, `z`) as UTC timezone metadata for timestamp fields - Infer strings with zero-offset timezone designators as UTC timestamps when date guessing is enabled