From 670afe76e1e65be53c998b8f9e265be57066fcc9 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:33:25 -0500 Subject: [PATCH] fix: resolve DST-ambiguous and nonexistent local times when casting naive timestamps to a named timezone Casting a `Timestamp(_, None)` to a `Timestamp(_, Some(tz))` means reading a wall clock time in `tz`. Around a daylight saving transition that reading is not always a single instant: the hour repeated by a "fall back" transition is ambiguous, and the hour skipped by a "spring forward" transition does not exist. arrow's cast kernel resolves the offset with `offset_from_local_datetime(..).single()`, which is `None` in both cases, so these casts fail with `Cannot cast timezone to different timezone` (or produce NULL under `TRY_CAST`). PostgreSQL and DuckDB resolve both deterministically, and DataFusion now does the same, in a new `datafusion_common::timezone_cast` module wired into the two DataFusion cast entry points (`ColumnarValue::cast_to` and `ScalarValue::cast_to_with_options`) for exactly that one pair of types: * ambiguous local times resolve to the later instant, i.e. the post-transition (standard) offset, so `2024-11-03T01:30:00` in `America/New_York` is `2024-11-03T01:30:00-05:00`; * nonexistent local times shift forward by the size of the gap, so `2024-03-10T02:30:00` in `America/New_York` is `2024-03-10T03:30:00-04:00`. Everything else, including the unit conversion, is still delegated to arrow. Closes https://github.com/apache/datafusion/issues/25084 Co-Authored-By: Claude Fable 5.1 --- datafusion/common/src/lib.rs | 1 + datafusion/common/src/scalar/mod.rs | 12 + datafusion/common/src/timezone_cast.rs | 393 ++++++++++++++++++ datafusion/expr-common/src/columnar_value.rs | 7 + .../datetime/cast_timestamp_dst.slt | 242 +++++++++++ 5 files changed, 655 insertions(+) create mode 100644 datafusion/common/src/timezone_cast.rs create mode 100644 datafusion/sqllogictest/test_files/datetime/cast_timestamp_dst.slt diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 2eebfe4963057..d0eb99592c56e 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -61,6 +61,7 @@ pub mod scalar; pub mod spans; pub mod stats; pub mod test_util; +pub mod timezone_cast; pub mod tree_node; pub mod types; pub mod utils; diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index bad526a3a2227..2d3ccf3111fce 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -4440,6 +4440,18 @@ impl ScalarValue { target_type, ) { crate::nested_struct::cast_column(&scalar_array, target_type, cast_options)? + } else if crate::timezone_cast::is_naive_to_timezone_cast( + &source_type, + target_type, + ) { + // Casting a timezone-naive timestamp into a timezone follows + // PostgreSQL/DuckDB semantics around daylight saving transitions, + // which differ from arrow's kernel. + crate::timezone_cast::cast_naive_timestamp_to_timezone( + &scalar_array, + target_type, + cast_options, + )? } else { cast_with_options(&scalar_array, target_type, cast_options)? }; diff --git a/datafusion/common/src/timezone_cast.rs b/datafusion/common/src/timezone_cast.rs new file mode 100644 index 0000000000000..09a3490646054 --- /dev/null +++ b/datafusion/common/src/timezone_cast.rs @@ -0,0 +1,393 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Casting a timezone-naive `Timestamp(_, None)` to a `Timestamp(_, Some(tz))`. +//! +//! Casting a naive (local) timestamp into a named timezone means interpreting a +//! wall clock reading in that timezone. Around a daylight saving transition two +//! wall clock readings are not a single instant: +//! +//! * **Ambiguous** — the hour repeated by a "fall back" transition. For example +//! `2024-11-03T01:30:00` occurs twice in `America/New_York`, once at +//! `-04:00` (EDT) and once at `-05:00` (EST). +//! * **Nonexistent** — the hour skipped by a "spring forward" transition. For +//! example `2024-03-10T02:30:00` never happens in `America/New_York`. +//! +//! [`arrow::compute::cast`] resolves the offset with +//! `offset_from_local_datetime(..).single()`, which is `None` for both cases, so +//! these casts fail with `Cannot cast timezone to different timezone` (or become +//! NULL when `CastOptions::safe` is set). PostgreSQL and DuckDB instead resolve +//! both deterministically, and this module implements their convention: +//! +//! * ambiguous — pick the **later** instant, i.e. the post-transition (standard) +//! offset, so `2024-11-03T01:30:00` in `America/New_York` is +//! `2024-11-03T01:30:00-05:00`; +//! * nonexistent — shift **forward** by the size of the gap, which is the same +//! as interpreting the wall clock reading with the pre-transition offset, so +//! `2024-03-10T02:30:00` in `America/New_York` is `2024-03-10T03:30:00-04:00`. +//! +//! Because this is the only pair of types where DataFusion's semantics differ +//! from the arrow kernel, DataFusion's cast entry points special-case exactly +//! that pair (see [`is_naive_to_timezone_cast`]) and delegate everything else, +//! including the unit conversion performed here, to arrow. + +use std::sync::Arc; + +use arrow::array::timezone::Tz; +use arrow::array::{Array, ArrayRef, AsArray, PrimitiveArray}; +use arrow::compute::CastOptions; +use arrow::compute::kernels::cast::cast_with_options; +use arrow::datatypes::{ + ArrowTimestampType, DataType, TimeUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, +}; +use arrow::error::ArrowError; +use arrow::temporal_conversions::as_datetime; +use chrono::{Duration, FixedOffset, NaiveDateTime, Offset, TimeZone}; + +use crate::Result; +use crate::error::_internal_err; + +/// Returns `true` if this is a cast from a timezone-naive timestamp to a +/// timestamp with a timezone, the one cast handled by +/// [`cast_naive_timestamp_to_timezone`]. +pub fn is_naive_to_timezone_cast(from: &DataType, to: &DataType) -> bool { + matches!( + (from, to), + ( + DataType::Timestamp(_, None), + DataType::Timestamp(_, Some(_)) + ) + ) +} + +/// Casts a timezone-naive timestamp array to `to_type`, a timestamp with a +/// timezone, resolving daylight saving ambiguities and gaps the way PostgreSQL +/// and DuckDB do (see the [module docs](self)). +/// +/// `to_type` must be a `Timestamp(_, Some(tz))` and `array` a +/// `Timestamp(_, None)`; use [`is_naive_to_timezone_cast`] to check. +pub fn cast_naive_timestamp_to_timezone( + array: &ArrayRef, + to_type: &DataType, + cast_options: &CastOptions<'static>, +) -> Result { + let ( + DataType::Timestamp(from_unit, None), + DataType::Timestamp(to_unit, Some(tz_str)), + ) = (array.data_type(), to_type) + else { + return _internal_err!( + "cast_naive_timestamp_to_timezone expects a naive timestamp source and a \ + timezone-aware timestamp target, got {} and {to_type}", + array.data_type() + ); + }; + + // Let arrow do the unit conversion (and its overflow handling) first, so + // that only the timezone adjustment is left to do here. + let array = if from_unit == to_unit { + Arc::clone(array) + } else { + cast_with_options(array, &DataType::Timestamp(*to_unit, None), cast_options)? + }; + + let tz: Tz = tz_str.parse()?; + + let adjusted: ArrayRef = match to_unit { + TimeUnit::Second => { + adjust::(&array, &tz, tz_str, cast_options)? + } + TimeUnit::Millisecond => { + adjust::(&array, &tz, tz_str, cast_options)? + } + TimeUnit::Microsecond => { + adjust::(&array, &tz, tz_str, cast_options)? + } + TimeUnit::Nanosecond => { + adjust::(&array, &tz, tz_str, cast_options)? + } + }; + + Ok(adjusted) +} + +/// Reinterprets every naive value of `array` as a wall clock reading in `tz`. +fn adjust( + array: &ArrayRef, + tz: &Tz, + tz_str: &Arc, + cast_options: &CastOptions<'static>, +) -> Result { + let array: &PrimitiveArray = array.as_primitive::(); + + let adjust = |value: i64| -> Option { + let local = as_datetime::(value)?; + let offset = resolve_local_offset(tz, local)?; + T::from_naive_datetime(local - offset, None) + }; + + let adjusted: PrimitiveArray = if cast_options.safe { + array.unary_opt::<_, T>(adjust) + } else { + array.try_unary::<_, T, _>(|value| { + adjust(value).ok_or_else(|| { + ArrowError::CastError( + "Cannot cast timezone to different timezone".to_string(), + ) + }) + })? + }; + + Ok(Arc::new(adjusted.with_timezone(Arc::clone(tz_str))) as ArrayRef) +} + +/// Resolves the UTC offset that a wall clock reading `local` has in `tz`, +/// following the PostgreSQL/DuckDB convention described in the +/// [module docs](self). +/// +/// Returns `None` only if the offset cannot be determined at all, which keeps +/// the error (or NULL, under `CastOptions::safe`) that arrow would produce. +fn resolve_local_offset(tz: &Tz, local: NaiveDateTime) -> Option { + match tz.offset_from_local_datetime(&local) { + chrono::LocalResult::Single(offset) => Some(offset.fix()), + // The wall clock reading happens twice, once before and once after a + // "fall back" transition. chrono returns the offsets in chronological + // order of the two instants, so the second one is the later instant. + chrono::LocalResult::Ambiguous(_earlier, later) => Some(later.fix()), + // The wall clock reading is inside a "spring forward" gap. Shifting it + // forward by the size of the gap is the same as interpreting it with + // the offset in effect before the transition, which we recover by + // probing a day earlier: no tzdb entry has two transitions within 24 + // hours of each other, so a day before a gap is always outside it. + chrono::LocalResult::None => tz + .offset_from_local_datetime(&(local - Duration::hours(24))) + .earliest() + .map(|offset| offset.fix()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{TimestampMillisecondArray, TimestampNanosecondArray}; + use arrow::datatypes::TimeUnit; + use chrono::NaiveDate; + + const NEW_YORK: &str = "America/New_York"; + const SYDNEY: &str = "Australia/Sydney"; + + fn local(s: &str) -> NaiveDateTime { + s.parse().unwrap() + } + + fn nanos(values: Vec>) -> ArrayRef { + Arc::new(TimestampNanosecondArray::from(values)) as ArrayRef + } + + /// Casts `values` (nanosecond naive timestamps) to `Timestamp(Nanosecond, Some(tz))` + /// and returns the resulting instants as epoch seconds. + fn cast_to_tz(values: Vec>, tz: &str) -> Vec> { + let array = nanos(values); + let to_type = DataType::Timestamp(TimeUnit::Nanosecond, Some(tz.into())); + let out = + cast_naive_timestamp_to_timezone(&array, &to_type, &CastOptions::default()) + .unwrap(); + assert_eq!(out.data_type(), &to_type); + out.as_primitive::() + .iter() + .map(|v| v.map(|v| v / 1_000_000_000)) + .collect() + } + + fn naive_nanos(s: &str) -> i64 { + local(s).and_utc().timestamp_nanos_opt().unwrap() + } + + #[test] + fn is_naive_to_timezone_cast_only_matches_that_pair() { + let naive = DataType::Timestamp(TimeUnit::Nanosecond, None); + let aware = DataType::Timestamp(TimeUnit::Second, Some("UTC".into())); + assert!(is_naive_to_timezone_cast(&naive, &aware)); + assert!(!is_naive_to_timezone_cast(&aware, &naive)); + assert!(!is_naive_to_timezone_cast(&aware, &aware)); + assert!(!is_naive_to_timezone_cast(&naive, &naive)); + assert!(!is_naive_to_timezone_cast(&DataType::Date32, &aware)); + } + + #[test] + fn resolves_unambiguous_local_times() { + let tz: Tz = NEW_YORK.parse().unwrap(); + // Eastern Daylight Time + assert_eq!( + resolve_local_offset(&tz, local("2024-11-01T00:00:00")).unwrap(), + FixedOffset::east_opt(-4 * 3600).unwrap() + ); + // Eastern Standard Time + assert_eq!( + resolve_local_offset(&tz, local("2024-12-01T00:00:00")).unwrap(), + FixedOffset::east_opt(-5 * 3600).unwrap() + ); + } + + #[test] + fn ambiguous_local_time_picks_the_later_instant() { + let ny: Tz = NEW_YORK.parse().unwrap(); + // 2024-11-03T01:30 happens twice: at -04:00 and then at -05:00. + assert_eq!( + resolve_local_offset(&ny, local("2024-11-03T01:30:00")).unwrap(), + FixedOffset::east_opt(-5 * 3600).unwrap() + ); + + let sydney: Tz = SYDNEY.parse().unwrap(); + // 2024-04-07T02:30 happens twice: at +11:00 and then at +10:00. + assert_eq!( + resolve_local_offset(&sydney, local("2024-04-07T02:30:00")).unwrap(), + FixedOffset::east_opt(10 * 3600).unwrap() + ); + } + + #[test] + fn nonexistent_local_time_shifts_forward_by_the_gap() { + let ny: Tz = NEW_YORK.parse().unwrap(); + // 2024-03-10T02:30 does not exist; the pre-transition offset is -05:00. + assert_eq!( + resolve_local_offset(&ny, local("2024-03-10T02:30:00")).unwrap(), + FixedOffset::east_opt(-5 * 3600).unwrap() + ); + + let sydney: Tz = SYDNEY.parse().unwrap(); + // 2024-10-06T02:30 does not exist; the pre-transition offset is +10:00. + assert_eq!( + resolve_local_offset(&sydney, local("2024-10-06T02:30:00")).unwrap(), + FixedOffset::east_opt(10 * 3600).unwrap() + ); + } + + #[test] + fn fixed_offset_timezones_are_never_ambiguous() { + let tz: Tz = "+08:00".parse().unwrap(); + for s in [ + "2024-11-03T01:30:00", + "2024-03-10T02:30:00", + "2024-01-01T00:00:00", + ] { + assert_eq!( + resolve_local_offset(&tz, local(s)).unwrap(), + FixedOffset::east_opt(8 * 3600).unwrap() + ); + } + } + + #[test] + fn cast_resolves_dst_boundaries() { + // Unambiguous: 2024-11-01T00:00 EDT (-04:00) == 2024-11-01T04:00Z. + let expected = NaiveDate::from_ymd_opt(2024, 11, 1) + .unwrap() + .and_hms_opt(4, 0, 0) + .unwrap() + .and_utc() + .timestamp(); + assert_eq!( + cast_to_tz(vec![Some(naive_nanos("2024-11-01T00:00:00"))], NEW_YORK), + vec![Some(expected)] + ); + + // Ambiguous: the later instant, 01:30-05:00 == 06:30Z == 1730615400. + // (The earlier candidate, 01:30-04:00, would be 1730611800.) + assert_eq!( + cast_to_tz(vec![Some(naive_nanos("2024-11-03T01:30:00"))], NEW_YORK), + vec![Some(1730615400)] + ); + + // Gap: shifted forward to 03:30-04:00 == 07:30Z == 1710055800. + assert_eq!( + cast_to_tz(vec![Some(naive_nanos("2024-03-10T02:30:00"))], NEW_YORK), + vec![Some(1710055800)] + ); + + // Sydney: ambiguous 02:30 resolves to +10:00 == 2024-04-06T16:30Z. + let expected = NaiveDate::from_ymd_opt(2024, 4, 6) + .unwrap() + .and_hms_opt(16, 30, 0) + .unwrap() + .and_utc() + .timestamp(); + assert_eq!( + cast_to_tz(vec![Some(naive_nanos("2024-04-07T02:30:00"))], SYDNEY), + vec![Some(expected)] + ); + + // Sydney: gap 02:30 shifts to 03:30+11:00 == 2024-10-05T16:30Z. + let expected = NaiveDate::from_ymd_opt(2024, 10, 5) + .unwrap() + .and_hms_opt(16, 30, 0) + .unwrap() + .and_utc() + .timestamp(); + assert_eq!( + cast_to_tz(vec![Some(naive_nanos("2024-10-06T02:30:00"))], SYDNEY), + vec![Some(expected)] + ); + } + + #[test] + fn cast_preserves_nulls() { + assert_eq!( + cast_to_tz( + vec![None, Some(naive_nanos("2024-11-03T01:30:00")), None], + NEW_YORK + ), + vec![None, Some(1730615400), None] + ); + } + + #[test] + fn cast_matches_arrow_when_the_local_time_is_unambiguous() { + let options = CastOptions::default(); + for (values, tz) in [ + (vec![Some(naive_nanos("2024-11-03T01:30:00"))], "+08:00"), + ( + vec![Some(naive_nanos("2024-06-15T12:00:00")), None], + NEW_YORK, + ), + ] { + let array = nanos(values); + let to_type = DataType::Timestamp(TimeUnit::Nanosecond, Some(tz.into())); + let ours = + cast_naive_timestamp_to_timezone(&array, &to_type, &options).unwrap(); + let arrows = cast_with_options(&array, &to_type, &options).unwrap(); + assert_eq!(&ours, &arrows, "timezone {tz}"); + } + } + + #[test] + fn cast_converts_the_time_unit_too() { + let array = nanos(vec![Some(naive_nanos("2024-11-03T01:30:00")), None]); + let to_type = DataType::Timestamp(TimeUnit::Millisecond, Some(NEW_YORK.into())); + let out = + cast_naive_timestamp_to_timezone(&array, &to_type, &CastOptions::default()) + .unwrap(); + assert_eq!(out.data_type(), &to_type); + let out = out + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(out.value(0), 1730615400 * 1_000); + assert!(out.is_null(1)); + } +} diff --git a/datafusion/expr-common/src/columnar_value.rs b/datafusion/expr-common/src/columnar_value.rs index ef9192c3569d9..f2e66c0f5c9eb 100644 --- a/datafusion/expr-common/src/columnar_value.rs +++ b/datafusion/expr-common/src/columnar_value.rs @@ -35,6 +35,7 @@ use datafusion_common::{ date_to_timestamp_multiplier, ensure_timestamp_in_bounds, timestamp_to_timestamp_multiplier, }, + timezone_cast::{cast_naive_timestamp_to_timezone, is_naive_to_timezone_cast}, }; use std::fmt; use std::sync::Arc; @@ -328,6 +329,12 @@ fn cast_array_by_name( if !cast_options.safe { ensure_temporal_array_timestamp_bounds(array, cast_type)?; } + // Casting a timezone-naive timestamp into a timezone follows + // PostgreSQL/DuckDB semantics around daylight saving transitions, which + // differ from arrow's kernel. + if is_naive_to_timezone_cast(array.data_type(), cast_type) { + return cast_naive_timestamp_to_timezone(array, cast_type, cast_options); + } Ok(kernels::cast::cast_with_options( array, cast_type, diff --git a/datafusion/sqllogictest/test_files/datetime/cast_timestamp_dst.slt b/datafusion/sqllogictest/test_files/datetime/cast_timestamp_dst.slt new file mode 100644 index 0000000000000..57d8090f54946 --- /dev/null +++ b/datafusion/sqllogictest/test_files/datetime/cast_timestamp_dst.slt @@ -0,0 +1,242 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +########## +## Casting a timezone-naive timestamp into a named timezone across a daylight +## saving transition. +## +## A local (wall clock) time is not always a single instant: +## * the hour repeated by a "fall back" transition is ambiguous, and +## * the hour skipped by a "spring forward" transition does not exist. +## +## Like PostgreSQL and DuckDB, DataFusion resolves the ambiguous case to the +## later instant (the post-transition, standard-time offset) and shifts a +## nonexistent local time forward by the size of the gap. +########## + +statement ok +SET datafusion.execution.time_zone = 'America/New_York'; + +# Unambiguous local time: 2024-11-01 is still EDT (-04:00) +query P +SELECT '2024-11-01T00:00:00'::timestamp::timestamptz; +---- +2024-11-01T00:00:00-04:00 + +# Ambiguous local time: 2024-11-03T01:30 occurs twice, at -04:00 and at -05:00. +# The later instant (-05:00) wins. +query P +SELECT '2024-11-03T01:30:00'::timestamp::timestamptz; +---- +2024-11-03T01:30:00-05:00 + +# Nonexistent local time: 2024-03-10T02:30 is skipped by the spring forward +# transition, so it is shifted forward by the one hour gap. +query P +SELECT '2024-03-10T02:30:00'::timestamp::timestamptz; +---- +2024-03-10T03:30:00-04:00 + +# The same three casts, as instants +query I +SELECT to_unixtime('2024-11-01T00:00:00'::timestamp::timestamptz); +---- +1730433600 + +# The earlier of the two candidate instants would be 1730611800 +query I +SELECT to_unixtime('2024-11-03T01:30:00'::timestamp::timestamptz); +---- +1730615400 + +# 2024-03-10T07:30:00Z +query I +SELECT to_unixtime('2024-03-10T02:30:00'::timestamp::timestamptz); +---- +1710055800 + +# The literal casts above are constant folded; a real column exercises the +# array cast kernel instead. +statement ok +CREATE TABLE t AS +SELECT arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, None)') AS ts; + +query TT +EXPLAIN SELECT ts::timestamptz FROM t; +---- +logical_plan +01)Projection: CAST(t.ts AS Timestamp(ns, "America/New_York")) +02)--TableScan: t projection=[ts] +physical_plan +01)ProjectionExec: expr=[CAST(ts@0 AS Timestamp(ns, "America/New_York")) as t.ts] +02)--DataSourceExec: partitions=1, partition_sizes=[1] + +query P +SELECT ts::timestamptz FROM t; +---- +2024-11-03T01:30:00-05:00 + +query I +SELECT to_unixtime(ts::timestamptz) FROM t; +---- +1730615400 + +statement ok +DROP TABLE t; + +# All four time units, ambiguous local time +query P +SELECT arrow_cast(arrow_cast('2024-11-03T01:30:00', 'Timestamp(Second, None)'), 'Timestamp(Second, Some("America/New_York"))'); +---- +2024-11-03T01:30:00-05:00 + +query P +SELECT arrow_cast(arrow_cast('2024-11-03T01:30:00', 'Timestamp(Millisecond, None)'), 'Timestamp(Millisecond, Some("America/New_York"))'); +---- +2024-11-03T01:30:00-05:00 + +query P +SELECT arrow_cast(arrow_cast('2024-11-03T01:30:00', 'Timestamp(Microsecond, None)'), 'Timestamp(Microsecond, Some("America/New_York"))'); +---- +2024-11-03T01:30:00-05:00 + +query P +SELECT arrow_cast(arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, None)'), 'Timestamp(Nanosecond, Some("America/New_York"))'); +---- +2024-11-03T01:30:00-05:00 + +# All four time units, nonexistent local time +query P +SELECT arrow_cast(arrow_cast('2024-03-10T02:30:00', 'Timestamp(Second, None)'), 'Timestamp(Second, Some("America/New_York"))'); +---- +2024-03-10T03:30:00-04:00 + +query P +SELECT arrow_cast(arrow_cast('2024-03-10T02:30:00', 'Timestamp(Millisecond, None)'), 'Timestamp(Millisecond, Some("America/New_York"))'); +---- +2024-03-10T03:30:00-04:00 + +query P +SELECT arrow_cast(arrow_cast('2024-03-10T02:30:00', 'Timestamp(Microsecond, None)'), 'Timestamp(Microsecond, Some("America/New_York"))'); +---- +2024-03-10T03:30:00-04:00 + +query P +SELECT arrow_cast(arrow_cast('2024-03-10T02:30:00', 'Timestamp(Nanosecond, None)'), 'Timestamp(Nanosecond, Some("America/New_York"))'); +---- +2024-03-10T03:30:00-04:00 + +# A cast that also changes the time unit +query P +SELECT arrow_cast(arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, None)'), 'Timestamp(Millisecond, Some("America/New_York"))'); +---- +2024-11-03T01:30:00-05:00 + +# TRY_CAST no longer yields NULL for these local times +query P +SELECT TRY_CAST('2024-11-03T01:30:00'::timestamp AS timestamptz); +---- +2024-11-03T01:30:00-05:00 + +query P +SELECT TRY_CAST('2024-03-10T02:30:00'::timestamp AS timestamptz); +---- +2024-03-10T03:30:00-04:00 + +# NULL in, NULL out +query P +SELECT arrow_cast(NULL, 'Timestamp(Nanosecond, None)')::timestamptz; +---- +NULL + +query P +SELECT TRY_CAST(arrow_cast(NULL, 'Timestamp(Nanosecond, None)') AS timestamptz); +---- +NULL + +########## +## Australia/Sydney, where daylight saving runs the other way round +########## + +statement ok +SET datafusion.execution.time_zone = 'Australia/Sydney'; + +# Ambiguous: 2024-04-07T02:30 occurs at +11:00 and then at +10:00 +query P +SELECT '2024-04-07T02:30:00'::timestamp::timestamptz; +---- +2024-04-07T02:30:00+10:00 + +query I +SELECT to_unixtime('2024-04-07T02:30:00'::timestamp::timestamptz); +---- +1712421000 + +# Nonexistent: 2024-10-06T02:30 is skipped, shifted forward by one hour +query P +SELECT '2024-10-06T02:30:00'::timestamp::timestamptz; +---- +2024-10-06T03:30:00+11:00 + +query I +SELECT to_unixtime('2024-10-06T02:30:00'::timestamp::timestamptz); +---- +1728145800 + +statement ok +CREATE TABLE t AS +SELECT arrow_cast('2024-10-06T02:30:00', 'Timestamp(Nanosecond, None)') AS ts; + +query P +SELECT ts::timestamptz FROM t; +---- +2024-10-06T03:30:00+11:00 + +statement ok +DROP TABLE t; + +########## +## A fixed offset never has a transition, so nothing changes there +########## + +statement ok +SET datafusion.execution.time_zone = '+08:00'; + +query P +SELECT '2024-11-03T01:30:00'::timestamp::timestamptz; +---- +2024-11-03T01:30:00+08:00 + +query P +SELECT '2024-03-10T02:30:00'::timestamp::timestamptz; +---- +2024-03-10T02:30:00+08:00 + +statement ok +CREATE TABLE t AS +SELECT arrow_cast('2024-03-10T02:30:00', 'Timestamp(Nanosecond, None)') AS ts; + +query P +SELECT ts::timestamptz FROM t; +---- +2024-03-10T02:30:00+08:00 + +statement ok +DROP TABLE t; + +statement ok +RESET datafusion.execution.time_zone;