From ed69900728bb0af8f8b6f3dc07904f1052817f16 Mon Sep 17 00:00:00 2001 From: Ben Chambers Date: Tue, 21 Jul 2026 13:01:25 -0700 Subject: [PATCH 1/6] fix(functions-nested): array_any_value returns NULL for empty list elements general_array_any_value handled null and all-null list elements, but a non-null *empty* (length-0) element fell through to the no-nulls branch, which unconditionally read values[start]. That returned the next element's value for an interior empty list (silently wrong data), and read out of bounds when start == values.len() (a trailing empty element), panicking with "range end index N out of range for slice of length N-1". The panic surfaced when the array_any_value output flowed into a hash RepartitionExec (e.g. used as an equi-join key): batches got sliced so an empty element landed at the end of a values buffer, tripping the out-of-bounds read on a spawned task. Guard the empty case explicitly: an empty list has no value to take, so the result is NULL. Sibling functions in this file are already safe (array_element bounds-checks the index against len; array_slice / pop_front / pop_back guard len == 0). Regression tests added at the kernel level (interior + trailing empty) and as sqllogictest cases. Signed-off-by: Ben Chambers --- datafusion/functions-nested/src/extract.rs | 55 ++++++++++++++++++- .../test_files/array/array_any_value.slt | 30 ++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index 900b408bffbba..b840448401e72 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -1062,13 +1062,23 @@ where for (row_index, offset_window) in array.offsets().windows(2).enumerate() { let start = offset_window[0]; + let end = offset_window[1]; - // array is null + // the list element is null if array.is_null(row_index) { mutable.try_extend_nulls(1)?; continue; } + // the list element is empty; there is no value to take, so the result + // is NULL. Without this guard the no-nulls branch below would read + // `values[start]`, which is either the next element (wrong value) or + // out of bounds when `start == values.len()` (panic). + if start == end { + mutable.try_extend_nulls(1)?; + continue; + } + let row_value = array.value(row_index); match row_value.nulls() { Some(row_nulls_buffer) => { @@ -1236,6 +1246,49 @@ mod tests { Ok(()) } + // An empty (length-0) list element that is not null must yield NULL, not + // the next element's value. Previously the no-nulls branch unconditionally + // read values[start], returning the wrong element for interior empty lists. + #[test] + fn test_array_any_value_empty_list_element() -> Result<()> { + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + // row 0 = [1], row 1 = [] (empty, non-null), row 2 = [2, 3] + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1, 1, 3])); + let field = Arc::new(Field::new("item", DataType::Int32, true)); + + let list_array = ListArray::new(field, offsets, values, None); + + let result = general_array_any_value(&list_array)?; + let result = result.as_any().downcast_ref::().unwrap(); + + assert_eq!(result.value(0), 1); + assert!(result.is_null(1)); // empty list -> NULL (previously read `2`) + assert_eq!(result.value(2), 2); + + Ok(()) + } + + // A trailing empty list element has start == values.len(); the old code did + // `extend(0, start, start + 1)` and panicked with an out-of-bounds slice. + #[test] + fn test_array_any_value_trailing_empty_list_element() -> Result<()> { + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); + // row 0 = [1], row 1 = [2], row 2 = [] (empty, non-null, start == len) + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1, 2, 2])); + let field = Arc::new(Field::new("item", DataType::Int32, true)); + + let list_array = ListArray::new(field, offsets, values, None); + + let result = general_array_any_value(&list_array)?; + let result = result.as_any().downcast_ref::().unwrap(); + + assert_eq!(result.value(0), 1); + assert_eq!(result.value(1), 2); + assert!(result.is_null(2)); // empty list -> NULL (previously panicked) + + Ok(()) + } + #[test] fn test_array_slice_list_view_basic() -> Result<()> { let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])); diff --git a/datafusion/sqllogictest/test_files/array/array_any_value.slt b/datafusion/sqllogictest/test_files/array/array_any_value.slt index 6579e88ac7dba..e7cb38fef82d1 100644 --- a/datafusion/sqllogictest/test_files/array/array_any_value.slt +++ b/datafusion/sqllogictest/test_files/array/array_any_value.slt @@ -145,6 +145,36 @@ select array_any_value(make_array(NULL, 1, 2, 3, 4, 5)), array_any_value(column1 1 41 1 51 +# array_any_value with empty (length-0) list elements +# A non-null but empty list must yield NULL, including a trailing empty element +# whose start offset equals the values length (previously read out of bounds +# and panicked the worker task). +statement ok +create table any_value_empty (id int, tags bigint[]) as values + (1, make_array(10)), + (2, cast(make_array() as bigint[])), + (3, make_array(20, 30)), + (4, cast(make_array() as bigint[])); + +query II +select id, array_any_value(tags) from any_value_empty order by id; +---- +1 10 +2 NULL +3 20 +4 NULL + +query II +select id, array_any_value(arrow_cast(tags, 'LargeList(Int64)')) from any_value_empty order by id; +---- +1 10 +2 NULL +3 20 +4 NULL + +statement ok +drop table any_value_empty; + # make_array with nulls query ??????? select make_array(make_array('a','b'), null), From f24ef4ff3b46f785c28d69ee4e0746bae323ea78 Mon Sep 17 00:00:00 2001 From: Ben Chambers Date: Wed, 22 Jul 2026 15:27:38 -0700 Subject: [PATCH 2/6] address review comments --- datafusion/functions-nested/src/extract.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index b840448401e72..b4e4c6cc8b0da 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -970,7 +970,7 @@ where #[user_doc( doc_section(label = "Array Functions"), - description = "Returns the first non-null element in the array.", + description = "Returns the first non-null element in the array. Returns NULL if the array is empty or all elements are NULL.", syntax_example = "array_any_value(array)", sql_example = r#"```sql > select array_any_value([NULL, 1, 2, 3]); @@ -1071,9 +1071,7 @@ where } // the list element is empty; there is no value to take, so the result - // is NULL. Without this guard the no-nulls branch below would read - // `values[start]`, which is either the next element (wrong value) or - // out of bounds when `start == values.len()` (panic). + // is NULL. if start == end { mutable.try_extend_nulls(1)?; continue; @@ -1247,8 +1245,7 @@ mod tests { } // An empty (length-0) list element that is not null must yield NULL, not - // the next element's value. Previously the no-nulls branch unconditionally - // read values[start], returning the wrong element for interior empty lists. + // the next element's value. #[test] fn test_array_any_value_empty_list_element() -> Result<()> { let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); @@ -1262,14 +1259,13 @@ mod tests { let result = result.as_any().downcast_ref::().unwrap(); assert_eq!(result.value(0), 1); - assert!(result.is_null(1)); // empty list -> NULL (previously read `2`) + assert!(result.is_null(1)); // empty list -> NULL assert_eq!(result.value(2), 2); Ok(()) } - // A trailing empty list element has start == values.len(); the old code did - // `extend(0, start, start + 1)` and panicked with an out-of-bounds slice. + // A trailing empty list element has start == values.len(). #[test] fn test_array_any_value_trailing_empty_list_element() -> Result<()> { let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); From a632d09f7fff15a435589cb049ac76cb422ee483 Mon Sep 17 00:00:00 2001 From: Ben Chambers Date: Wed, 22 Jul 2026 15:33:36 -0700 Subject: [PATCH 3/6] another review comment --- datafusion/sqllogictest/test_files/array/array_any_value.slt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datafusion/sqllogictest/test_files/array/array_any_value.slt b/datafusion/sqllogictest/test_files/array/array_any_value.slt index e7cb38fef82d1..c8976e8493261 100644 --- a/datafusion/sqllogictest/test_files/array/array_any_value.slt +++ b/datafusion/sqllogictest/test_files/array/array_any_value.slt @@ -147,8 +147,7 @@ select array_any_value(make_array(NULL, 1, 2, 3, 4, 5)), array_any_value(column1 # array_any_value with empty (length-0) list elements # A non-null but empty list must yield NULL, including a trailing empty element -# whose start offset equals the values length (previously read out of bounds -# and panicked the worker task). +# whose start offset equals the values length statement ok create table any_value_empty (id int, tags bigint[]) as values (1, make_array(10)), From 882764cc16b97a3490a479ca1aa7b208097fe356 Mon Sep 17 00:00:00 2001 From: Ben Chambers Date: Wed, 22 Jul 2026 19:38:34 -0700 Subject: [PATCH 4/6] update user guide --- docs/source/user-guide/sql/scalar_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index cb748f57d9967..7047265888e1a 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -3430,7 +3430,7 @@ any_match(array, predicate) ### `array_any_value` -Returns the first non-null element in the array. +Returns the first non-null element in the array. Returns NULL if the array is empty or all elements are NULL. ```sql array_any_value(array) From 6d80d519417ce88ecf1263ef2fba835ac7cfd46e Mon Sep 17 00:00:00 2001 From: Ben Chambers Date: Wed, 22 Jul 2026 22:20:36 -0700 Subject: [PATCH 5/6] remove tests covered by slt --- datafusion/functions-nested/src/extract.rs | 41 ---------------------- 1 file changed, 41 deletions(-) diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index b4e4c6cc8b0da..c36a30d07cb14 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -1244,47 +1244,6 @@ mod tests { Ok(()) } - // An empty (length-0) list element that is not null must yield NULL, not - // the next element's value. - #[test] - fn test_array_any_value_empty_list_element() -> Result<()> { - let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); - // row 0 = [1], row 1 = [] (empty, non-null), row 2 = [2, 3] - let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1, 1, 3])); - let field = Arc::new(Field::new("item", DataType::Int32, true)); - - let list_array = ListArray::new(field, offsets, values, None); - - let result = general_array_any_value(&list_array)?; - let result = result.as_any().downcast_ref::().unwrap(); - - assert_eq!(result.value(0), 1); - assert!(result.is_null(1)); // empty list -> NULL - assert_eq!(result.value(2), 2); - - Ok(()) - } - - // A trailing empty list element has start == values.len(). - #[test] - fn test_array_any_value_trailing_empty_list_element() -> Result<()> { - let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2])); - // row 0 = [1], row 1 = [2], row 2 = [] (empty, non-null, start == len) - let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1, 2, 2])); - let field = Arc::new(Field::new("item", DataType::Int32, true)); - - let list_array = ListArray::new(field, offsets, values, None); - - let result = general_array_any_value(&list_array)?; - let result = result.as_any().downcast_ref::().unwrap(); - - assert_eq!(result.value(0), 1); - assert_eq!(result.value(1), 2); - assert!(result.is_null(2)); // empty list -> NULL (previously panicked) - - Ok(()) - } - #[test] fn test_array_slice_list_view_basic() -> Result<()> { let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])); From 7b93d47c41604c5afaa7bab5193f590ba1294c6d Mon Sep 17 00:00:00 2001 From: Ben Chambers Date: Thu, 23 Jul 2026 13:01:00 -0700 Subject: [PATCH 6/6] doc update --- datafusion/functions-nested/src/extract.rs | 2 +- docs/source/user-guide/sql/scalar_functions.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index c36a30d07cb14..b1c22822dfdc7 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -970,7 +970,7 @@ where #[user_doc( doc_section(label = "Array Functions"), - description = "Returns the first non-null element in the array. Returns NULL if the array is empty or all elements are NULL.", + description = "Returns the first non-null element in the array. Returns NULL if the array is empty or NULL.", syntax_example = "array_any_value(array)", sql_example = r#"```sql > select array_any_value([NULL, 1, 2, 3]); diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 7047265888e1a..a285b9e5f5cff 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -3430,7 +3430,7 @@ any_match(array, predicate) ### `array_any_value` -Returns the first non-null element in the array. Returns NULL if the array is empty or all elements are NULL. +Returns the first non-null element in the array. Returns NULL if the array is empty or NULL. ```sql array_any_value(array)