diff --git a/Changes.md b/Changes.md index 94da8b24..029a14af 100644 --- a/Changes.md +++ b/Changes.md @@ -1,5 +1,17 @@ # Change log +## Development + +- Accept zero-offset timezone designators (`+00:00`, `-00:00`, `+0000`, `-0000`, + `Z`, `z`) as UTC timezone metadata for timestamp fields +- Infer strings with zero-offset timezone designators as UTC timestamps when + date guessing is enabled + +### Thanks + +- [@ryzhyk](https://github.com/ryzhyk) added support for zero-offset timezone + designators ([#320](https://github.com/chmp/serde_arrow/pull/320)) + ## 0.15.0-rc.1 Breaking changes: diff --git a/serde_arrow/src/internal/chrono.rs b/serde_arrow/src/internal/chrono.rs index f2ce2c4c..44e1469f 100644 --- a/serde_arrow/src/internal/chrono.rs +++ b/serde_arrow/src/internal/chrono.rs @@ -29,6 +29,16 @@ pub fn matches_naive_time(s: &str) -> bool { parsing::match_naive_time(s).matches() } +/// Check whether an Arrow timezone string denotes UTC (a zero offset). +/// +/// Arrow encodes every timezone-aware timestamp as a UTC instant; the timezone +/// string only records how to display it. Recognize the name `UTC` and the fixed +/// zero-offset designators. +pub fn is_utc_timezone(tz: &str) -> bool { + tz.eq_ignore_ascii_case("utc") + || matches!(tz, "Z" | "z" | "+00:00" | "-00:00" | "+0000" | "-0000") +} + /// Parse `s` as a span pub fn parse_span(s: &str) -> Result> { parsing::match_span(s).into_result("Span") @@ -345,7 +355,7 @@ mod parsing { /// /// Note: this function is more permissive than some libraries (e.g., jiff) pub fn match_utc_timezone(s: &str) -> Result<(&str, &str), &str> { - for prefix in ["Z", "+0000", "+00:00"] { + for prefix in ["Z", "z", "+0000", "-0000", "+00:00", "-00:00"] { if let Some((prefix, rest)) = split_prefix(s, prefix) { return Ok((rest, prefix)); } diff --git a/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs b/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs index ea188f4d..e61e8bfa 100644 --- a/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs +++ b/serde_arrow/src/internal/deserialization/timestamp_deserializer.rs @@ -6,6 +6,7 @@ use marrow::{ use serde::de::Visitor; use crate::internal::{ + chrono::is_utc_timezone, error::{fail, set_default, try_, Context, ContextSupport, Result}, utils::array_view_ext::ViewAccess, }; @@ -75,9 +76,9 @@ impl<'a> TimestampDeserializer<'a> { fn is_utc_timestamp(timezone: Option<&str>) -> Result { match timezone { - Some(tz) if tz.to_lowercase() == "utc" => Ok(true), - Some(tz) => fail!("unsupported timezone: {} is not supported", tz), None => Ok(false), + Some(tz) if is_utc_timezone(tz) => Ok(true), + Some(tz) => fail!("unsupported timezone: {} is not supported", tz), } } diff --git a/serde_arrow/src/internal/schema/mod.rs b/serde_arrow/src/internal/schema/mod.rs index 2581d413..d6adca55 100644 --- a/serde_arrow/src/internal/schema/mod.rs +++ b/serde_arrow/src/internal/schema/mod.rs @@ -108,7 +108,8 @@ pub trait SchemaLike: Sized + Sealed { /// - date objects: `"Date32"`, `"Date64"` /// - datetime objects: `"Timestamp(unit, optional_timezone)"` with `unit` being one of /// `Second`, `Millisecond`, `Microsecond`, `Nanosecond` and `optional_timezone` being either - /// `None` or `Some("Utc")`. + /// `None` or a UTC zero-offset designator (`Some("Utc")`, `Some("Z")`, `Some("+00:00")`, + /// `Some("-00:00")`, `Some("+0000")`, or `Some("-0000")`). /// - time objects: `"Time32(unit)"`, `"Time64(unit)"` with unit being one of `Second`, /// `Millisecond`, `Microsecond`, `Nanosecond`. /// - durations: `"Duration(unit)"` with unit being one of `Second`, `Millisecond`, diff --git a/serde_arrow/src/internal/serialization/timestamp_builder.rs b/serde_arrow/src/internal/serialization/timestamp_builder.rs index 2f4670b2..1d8c2bb3 100644 --- a/serde_arrow/src/internal/serialization/timestamp_builder.rs +++ b/serde_arrow/src/internal/serialization/timestamp_builder.rs @@ -7,6 +7,7 @@ use marrow::{ use serde::Serialize; use crate::internal::{ + chrono::is_utc_timezone, error::{fail, set_default, Context, ContextSupport, Result}, serialization::utils::impl_serializer, utils::array_ext::{ArrayExt, ScalarArrayExt}, @@ -88,7 +89,7 @@ impl TimestampBuilder { fn is_utc_tz(tz: Option<&str>) -> Result { match tz { None => Ok(false), - Some(tz) if tz.to_uppercase() == "UTC" => Ok(true), + Some(tz) if is_utc_timezone(tz) => Ok(true), Some(tz) => fail!("timezone {tz} is not supported"), } } diff --git a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs index 2e31adc2..efde28cb 100644 --- a/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs +++ b/serde_arrow/src/test_with_arrow/impls/arrow_timestamp.rs @@ -184,6 +184,36 @@ mod schema_tracing { .check_nulls(&[&[false, true, false]]); } + #[test] + fn utc_zero_offset_designators_as_timestamp_tracing() { + let items = [ + Item(String::from("2025-01-20T19:30:42+0000")), + Item(String::from("2025-01-20T19:30:42+00:00")), + Item(String::from("2025-01-20T19:30:42-0000")), + Item(String::from("2025-01-20T19:30:42-00:00")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42z")), + ]; + let expected = [ + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42Z")), + ]; + + Test::new() + .with_schema(json!([{ + "name": "item", + "data_type": "Timestamp(Millisecond, Some(\"UTC\"))", + }])) + .trace_schema_from_samples(&items, TracingOptions::default().guess_dates(true)) + .serialize(&items) + .deserialize(&expected) + .check_nulls(&[&[false, false, false, false, false, false]]); + } + #[test] fn utc_tracing_string_only_with_invalid() { let items = [ @@ -269,3 +299,118 @@ mod schema_tracing { .check_nulls(&[&[false, false]]); } } + +/// A timezone-aware Arrow timestamp is a UTC instant, so every zero-offset +/// timezone designator must deserialize. Iceberg tags its `timestamptz` columns +/// with the fixed offset `+00:00`. +mod timezone_utc_designators { + use super::*; + + const UTC_DESIGNATORS: &[&str] = + &["+00:00", "-00:00", "+0000", "-0000", "Z", "z", "UTC", "utc"]; + + #[test] + fn deserializes_every_utc_designator_as_zulu() { + for tz in UTC_DESIGNATORS { + let array = Array::Timestamp(TimestampArray { + unit: TimeUnit::Microsecond, + timezone: Some(String::from(*tz)), + validity: None, + values: vec![zulu_micros()], + }); + let view = array.as_view(); + let deserializer = Deserializer::from_marrow(&[tz_field(tz)], &[view]) + .unwrap_or_else(|e| panic!("timezone {tz:?} rejected: {e}")); + let actual = Vec::>::deserialize(deserializer).unwrap(); + + assert_eq!( + actual, + [Item(String::from("2025-01-20T19:30:42Z"))], + "timezone {tz:?}" + ); + } + } + + /// Tagging the schema with a zero-offset designator sets the builder to UTC, + /// so a `Z`-suffixed string parses instead of being rejected as non-naive. + #[test] + fn serializes_every_utc_designator_from_zulu_string() { + for tz in UTC_DESIGNATORS { + let mut builder = ArrayBuilder::from_marrow(&[tz_field(tz)]) + .unwrap_or_else(|e| panic!("timezone {tz:?} rejected: {e}")); + [Item(String::from("2025-01-20T19:30:42Z"))] + .serialize(Serializer::new(&mut builder)) + .unwrap_or_else(|e| panic!("timezone {tz:?}: {e}")); + + let arrays = builder.to_marrow().unwrap(); + let [array] = <[_; 1]>::try_from(arrays).unwrap(); + let Array::Timestamp(array) = array else { + panic!("timezone {tz:?}: expected a timestamp array"); + }; + + assert_eq!(array.values, [zulu_micros()], "timezone {tz:?}"); + assert_eq!(array.timezone.as_deref(), Some(*tz), "timezone {tz:?}"); + } + } + + #[test] + fn serializes_zero_offset_designators_from_string_values() { + let items = [ + Item(String::from("2025-01-20T19:30:42+0000")), + Item(String::from("2025-01-20T19:30:42+00:00")), + Item(String::from("2025-01-20T19:30:42-0000")), + Item(String::from("2025-01-20T19:30:42-00:00")), + Item(String::from("2025-01-20T19:30:42Z")), + Item(String::from("2025-01-20T19:30:42z")), + ]; + let mut builder = ArrayBuilder::from_marrow(&[tz_field("UTC")]).unwrap(); + items + .serialize(Serializer::new(&mut builder)) + .unwrap_or_else(|e| panic!("{e}")); + + let arrays = builder.to_marrow().unwrap(); + let [array] = <[_; 1]>::try_from(arrays).unwrap(); + let Array::Timestamp(array) = array else { + panic!("expected a timestamp array"); + }; + + assert_eq!(array.values, vec![zulu_micros(); items.len()]); + assert_eq!(array.timezone.as_deref(), Some("UTC")); + } + + #[test] + fn rejects_non_utc_offset() { + let array = Array::Timestamp(TimestampArray { + unit: TimeUnit::Microsecond, + timezone: Some(String::from("+01:00")), + validity: None, + values: vec![0], + }); + let view = array.as_view(); + let err = match Deserializer::from_marrow(&[tz_field("+01:00")], &[view]) { + Ok(_) => panic!("expected the +01:00 offset to be rejected"), + Err(err) => err, + }; + assert!( + err.to_string().contains("+01:00"), + "unexpected error: {err}" + ); + } + + /// `2025-01-20T19:30:42Z` as microseconds since the epoch. + fn zulu_micros() -> i64 { + NaiveDateTime::parse_from_str("2025-01-20T19:30:42", "%Y-%m-%dT%H:%M:%S") + .unwrap() + .and_utc() + .timestamp_micros() + } + + fn tz_field(tz: &str) -> Field { + Field { + name: String::from("item"), + data_type: DataType::Timestamp(TimeUnit::Microsecond, Some(String::from(tz))), + nullable: false, + metadata: Default::default(), + } + } +}