diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index 6228370c955a9..36963caeb57ed 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -29,7 +29,8 @@ runs: shell: bash run: | RETRY=("ci/scripts/retry" timeout 120) - "${RETRY[@]}" apt-get update + rm -f /etc/apt/sources.list.d/google-chrome.list + "${RETRY[@]}" apt-get update || true "${RETRY[@]}" apt-get install -y protobuf-compiler - name: Setup Rust toolchain shell: bash @@ -59,4 +60,6 @@ runs: # remove Android library: about 7.8GB (host /usr/local/lib/android) rm -rf /host/usr/local/lib/android || true echo "Disk space after cleanup:" - df -h \ No newline at end of file + df -h + + diff --git a/.github/workflows/breaking_changes_detector.yml b/.github/workflows/breaking_changes_detector.yml index 805c42dcb5c83..01fa666ab7bdb 100644 --- a/.github/workflows/breaking_changes_detector.yml +++ b/.github/workflows/breaking_changes_detector.yml @@ -84,7 +84,8 @@ jobs: - name: Install Protobuf Compiler if: steps.changed_crates.outputs.packages != '' run: | - sudo apt-get update + sudo rm -f /etc/apt/sources.list.d/google-chrome.list + sudo apt-get update || true sudo apt-get install -y protobuf-compiler - name: Install cargo-semver-checks diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 21c4223dacacc..ce985284d0327 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -50,7 +50,8 @@ jobs: - name: Install Graphviz run: | set -x - sudo apt-get update + sudo rm -f /etc/apt/sources.list.d/google-chrome.list + sudo apt-get update || true sudo apt-get install -y graphviz - name: Install cargo-depgraph uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index 4362323ec97f0..9ae92a6e0995d 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -56,7 +56,8 @@ jobs: - name: Install Graphviz run: | set -x - sudo apt-get update + sudo rm -f /etc/apt/sources.list.d/google-chrome.list + sudo apt-get update || true sudo apt-get install -y graphviz - name: Install cargo-depgraph uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index a6e303e3d6ff4..021137b572b6b 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -79,7 +79,8 @@ jobs: rustup toolchain install - name: Install Protobuf Compiler run: | - sudo apt-get update + sudo rm -f /etc/apt/sources.list.d/google-chrome.list + sudo apt-get update || true sudo apt-get install -y protobuf-compiler # For debugging, test binaries can be large. - name: Show available disk space @@ -144,7 +145,8 @@ jobs: # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive - name: Install protobuf compiler run: | - apt-get update && apt-get install -y protobuf-compiler + apt-get update || true + apt-get install -y protobuf-compiler - name: Run sqllogictest run: | cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 518e4d20eecb8..c14af57a02842 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -463,7 +463,8 @@ jobs: rustup target add wasm32-unknown-unknown - name: Install dependencies run: | - sudo apt-get update -qq + sudo rm -f /etc/apt/sources.list.d/google-chrome.list + sudo apt-get update -qq || true sudo apt-get install -y -qq clang - name: Setup wasm-pack uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 diff --git a/datafusion/expr-common/src/casts.rs b/datafusion/expr-common/src/casts.rs index 3518c02772672..e93606e02299d 100644 --- a/datafusion/expr-common/src/casts.rs +++ b/datafusion/expr-common/src/casts.rs @@ -113,6 +113,16 @@ fn is_date_type(data_type: &DataType) -> bool { /// `Date64` carrying sub-day milliseconds would lose them. This is not a licence to /// drop them - [`try_cast_numeric_literal`] returns `None` for a `Date64` value not /// divisible by 86_400_000, so an inexact `Date64` -> `Date32` fold never happens. +/// +/// **Timezone Shifts:** +/// Conversions between timezone-naive and timezone-aware timestamps are +/// mathematically bijective (shifting the physical value by the timezone offset), +/// rather than many-to-one lossy. However, we return `true` here to block unwrapping +/// as an intentionally conservative guard. If we returned `false`, `unwrap_cast_in_comparison` +/// would strip the cast but fail to shift the underlying literal, returning incorrect +/// query results. (A robust alternative would be to allow the unwrap and shift the literal, +/// preserving pushdown and pruning.) Only UTC-equivalent timezones (where the shift is +/// exactly zero) are allowed to bypass this guard. fn is_lossy_temporal_cast(from_type: &DataType, to_type: &DataType) -> bool { if from_type == to_type { return false; @@ -120,10 +130,37 @@ fn is_lossy_temporal_cast(from_type: &DataType, to_type: &DataType) -> bool { if is_date_type(from_type) && is_date_type(to_type) { return false; } + if let (DataType::Timestamp(_, from_tz), DataType::Timestamp(_, to_tz)) = + (from_type, to_type) + { + match (from_tz, to_tz) { + (Some(tz), None) | (None, Some(tz)) + if !is_zero_offset_timezone(tz.as_ref()) => + { + return true; + } + _ => {} + } + } (is_date_type(from_type) && to_type.is_temporal()) || (is_date_type(to_type) && from_type.is_temporal()) } +/// Returns true if the timezone is known to have a fixed zero offset from UTC. +/// +/// This is used to determine if a cast between a timezone-aware and timezone-naive +/// timestamp is lossy. If the timezone is strictly UTC-equivalent, the cast is +/// a lossless re-labeling of the integer value. +fn is_zero_offset_timezone(tz: &str) -> bool { + match tz { + // Standard UTC identifiers + "UTC" | "Etc/UTC" | "GMT" | "Etc/GMT" | "Greenwich" | "Z" => true, + // Common fixed offset zero strings parsed by Arrow + "+00:00" | "-00:00" | "+0:00" | "-0:00" => true, + _ => false, + } +} + /// Returns true when casting a timestamp from `from_type` to `to_type` loses /// timestamp precision. /// @@ -998,6 +1035,31 @@ mod tests { assert!(is_lossy_temporal_cast(&ts, &DataType::Date32)); } + #[test] + fn test_is_lossy_temporal_cast_timestamp_tz() { + let ts_naive = DataType::Timestamp(TimeUnit::Millisecond, None); + let ts_utc = DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())); + let ts_etc_utc = + DataType::Timestamp(TimeUnit::Millisecond, Some("Etc/UTC".into())); + let ts_gmt = DataType::Timestamp(TimeUnit::Millisecond, Some("GMT".into())); + let ts_sgt = + DataType::Timestamp(TimeUnit::Millisecond, Some("Asia/Singapore".into())); + + // Naive <-> UTC is NOT lossy (UTC offset is 0, so literal cast is exact) + assert!(!is_lossy_temporal_cast(&ts_naive, &ts_utc)); + assert!(!is_lossy_temporal_cast(&ts_utc, &ts_naive)); + assert!(!is_lossy_temporal_cast(&ts_naive, &ts_etc_utc)); + assert!(!is_lossy_temporal_cast(&ts_naive, &ts_gmt)); + + // Naive <-> Non-UTC is lossy because it ignores session timezone + assert!(is_lossy_temporal_cast(&ts_naive, &ts_sgt)); + assert!(is_lossy_temporal_cast(&ts_sgt, &ts_naive)); + + // Tz-aware <-> Tz-aware is not lossy (both are UTC under the hood) + assert!(!is_lossy_temporal_cast(&ts_utc, &ts_sgt)); + assert!(!is_lossy_temporal_cast(&ts_sgt, &ts_utc)); + } + #[test] fn test_timestamp_precision_narrowing_cast() { let ts_ns = DataType::Timestamp(TimeUnit::Nanosecond, None); diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index d73bc6eb06de8..3ed23323bba12 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -4301,6 +4301,10 @@ SELECT column1 FROM t_utc WHERE column1 < '2024-02-01T00:00:00' AT TIME ZONE 'Am query P SELECT column1 FROM t_europe WHERE column1 = '2024-01-31T16:00:01' AT TIME ZONE 'America/Los_Angeles'; ---- + +query P +SELECT column1 FROM t_europe WHERE column1 = '2024-01-31T15:00:01' AT TIME ZONE 'America/Los_Angeles'; +---- 2024-02-01T00:00:01+01:00 query P @@ -5529,3 +5533,75 @@ query P SELECT date_bin(NULL, TIMESTAMP '2023-01-01 12:30:00', TIMESTAMP '2023-01-01 12:00:00') ---- NULL + +# Issue 25095: Optimizer incorrectly unwrapping timestamp cast when session timezone is not UTC +statement ok +set datafusion.execution.time_zone = 'Asia/Singapore'; + +statement ok +create table t_25095 as select TIMESTAMP '2024-11-01T00:00:00' as ts; + +statement ok +create table u_25095 as select '2024-10-31T16:00:00Z'::timestamptz as tstz; + +# 2024-11-01 00:00 in Singapore is 2024-10-31 16:00 UTC +query I +select count(*) from t_25095 where ts::timestamptz = '2024-10-31T16:00:00Z'::timestamptz; +---- +1 + +query I +select count(*) from t_25095 where ts::timestamptz = '2024-11-01T00:00:00Z'::timestamptz; +---- +0 + +# the same rewrite occurs for an implicit coercion +query I +select count(*) from t_25095 where ts = '2024-10-31T16:00:00Z'::timestamptz; +---- +1 + +# control: a column against a column, thus the optimizer unwraps nothing +query I +select count(*) from t_25095, u_25095 where t_25095.ts::timestamptz = u_25095.tstz; +---- +1 + +# A timezone-aware column against a timezone-naive literal +query I +select count(*) from u_25095 where tstz = TIMESTAMP '2024-11-01T00:00:00'; +---- +1 + +# Set session timezone back to UTC and demonstrate that the cast IS unwrapped +statement ok +set datafusion.execution.time_zone = 'UTC'; + +# The explain output should show that the cast s::timestamptz has been removed +# because we allow unwrap_cast_in_comparison for UTC offsets +query TT +EXPLAIN select count(*) from t_25095 where ts::timestamptz = '2024-11-01T00:00:00Z'::timestamptz; +---- +logical_plan +01)Projection: count(Int64(1)) AS count(*) +02)--Aggregate: groupBy=[[]], aggr=[[count(Int64(1))]] +03)----Projection: +04)------Filter: t_25095.ts = TimestampNanosecond(1730419200000000000, None) +05)--------TableScan: t_25095 projection=[ts] +physical_plan +01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)] +02)--AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] +03)----CoalescePartitionsExec +04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] +05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +06)----------FilterExec: ts@0 = 1730419200000000000, projection=[] +07)------------DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +drop table t_25095; + +statement ok +drop table u_25095; + +statement ok +RESET datafusion.execution.time_zone;