Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions cpp/include/cudf_test/nanoarrow_utils.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,69 @@ struct generated_test_data {
std::vector<uint8_t> list_validity;
};

// Build a struct schema carrying a single fixed_size_list<int64>[width] child named "a".
// ArrowSchemaInitFromType cannot initialize a fixed-size-list schema, and
// ArrowSchemaSetTypeFixedSize leaves the allocated child type unset.
inline nanoarrow::UniqueSchema make_struct_fixed_size_list_int64_schema(int32_t width,
bool nullable = false)
{
nanoarrow::UniqueSchema schema;
ArrowSchemaInit(schema.get());
NANOARROW_THROW_NOT_OK(ArrowSchemaSetTypeStruct(schema.get(), 1));

NANOARROW_THROW_NOT_OK(
ArrowSchemaSetTypeFixedSize(schema->children[0], NANOARROW_TYPE_FIXED_SIZE_LIST, width));
NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "a"));
schema->children[0]->flags = nullable ? ARROW_FLAG_NULLABLE : 0;

NANOARROW_THROW_NOT_OK(
ArrowSchemaSetType(schema->children[0]->children[0], NANOARROW_TYPE_INT64));
NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0]->children[0], "element"));
schema->children[0]->children[0]->flags = 0;

return schema;
}

// Build an array matching make_struct_fixed_size_list_int64_schema. Fixed-size-list arrays
// carry no offsets buffer; validity belongs to the list child of the outer struct.
inline nanoarrow::UniqueArray make_struct_fixed_size_list_int64_array(
ArrowSchema* schema,
std::vector<int64_t> const& values,
int64_t num_rows,
std::vector<uint8_t> const& validity = {})
{
nanoarrow::UniqueArray array;
NANOARROW_THROW_NOT_OK(ArrowArrayInitFromSchema(array.get(), schema, nullptr));
array->length = num_rows;
array->null_count = 0;

auto* list_array = array->children[0];
list_array->length = num_rows;
list_array->null_count = 0;
if (!validity.empty()) {
ArrowBitmap bitmap;
ArrowBitmapInit(&bitmap);
NANOARROW_THROW_NOT_OK(ArrowBitmapReserve(&bitmap, validity.size()));
ArrowBitmapAppendInt8Unsafe(
&bitmap, reinterpret_cast<int8_t const*>(validity.data()), validity.size());
ArrowArraySetValidityBitmap(list_array, &bitmap);
list_array->null_count =
num_rows -
ArrowBitCountSet(ArrowArrayValidityBitmap(list_array)->buffer.data, 0, validity.size());
}

auto* values_array = list_array->children[0];
NANOARROW_THROW_NOT_OK(ArrowBufferAppend(ArrowArrayBuffer(values_array, 1),
reinterpret_cast<void const*>(values.data()),
values.size() * sizeof(int64_t)));
values_array->length = values.size();
values_array->null_count = 0;

NANOARROW_THROW_NOT_OK(
ArrowArrayFinishBuilding(array.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr));
return array;
}

// no-op allocator/deallocator to set into ArrowArray buffers that we don't
// want to own their buffers.
static ArrowBufferAllocator noop_alloc = (struct ArrowBufferAllocator){
Expand Down
21 changes: 21 additions & 0 deletions cpp/src/interop/arrow_data_structures.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,23 @@

namespace cudf::interop {

namespace {

bool contains_fixed_size_list(ArrowSchema const& schema)
{
ArrowSchemaView schema_view;
NANOARROW_THROW_NOT_OK(ArrowSchemaViewInit(&schema_view, &schema, nullptr));
if (schema_view.type == NANOARROW_TYPE_FIXED_SIZE_LIST) { return true; }

for (auto i = 0; i < schema.n_children; ++i) {
if (contains_fixed_size_list(*schema.children[i])) { return true; }
}

return schema.dictionary != nullptr && contains_fixed_size_list(*schema.dictionary);
}

} // namespace

/**
* @brief A wrapper around ArrowDeviceArray data used for flexible lifetime management.
*
Expand Down Expand Up @@ -66,6 +83,10 @@ struct arrow_array_container {
rmm::cuda_stream_view stream,
rmm::device_async_resource_ref mr)
{
CUDF_EXPECTS(!contains_fixed_size_list(schema_),
"Importing fixed-size-list columns through owning Arrow device-array wrappers is "
"not supported",
cudf::data_type_error);
switch (input_.device_type) {
case ARROW_DEVICE_CUDA:
case ARROW_DEVICE_CUDA_HOST:
Expand Down
67 changes: 65 additions & 2 deletions cpp/src/interop/arrow_utilities.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand All @@ -11,6 +11,9 @@

#include <nanoarrow/nanoarrow.h>

#include <limits>
#include <stdexcept>

namespace cudf {
namespace detail {
data_type arrow_to_cudf_type(ArrowSchemaView const* arrow_view)
Expand All @@ -33,7 +36,8 @@ data_type arrow_to_cudf_type(ArrowSchemaView const* arrow_view)
case NANOARROW_TYPE_STRING_VIEW:
case NANOARROW_TYPE_LARGE_STRING: return data_type(type_id::STRING);
case NANOARROW_TYPE_LIST:
case NANOARROW_TYPE_LARGE_LIST: return data_type(type_id::LIST);
case NANOARROW_TYPE_LARGE_LIST:
case NANOARROW_TYPE_FIXED_SIZE_LIST: return data_type(type_id::LIST);
case NANOARROW_TYPE_DICTIONARY: return data_type(type_id::DICTIONARY32);
case NANOARROW_TYPE_STRUCT: return data_type(type_id::STRUCT);
case NANOARROW_TYPE_TIMESTAMP: {
Expand Down Expand Up @@ -62,6 +66,65 @@ data_type arrow_to_cudf_type(ArrowSchemaView const* arrow_view)
}
}

bool is_fixed_size_list(ArrowSchemaView const* arrow_view)
{
return arrow_view->type == NANOARROW_TYPE_FIXED_SIZE_LIST;
}

int32_t fixed_size_list_width(ArrowSchemaView const* arrow_view)
{
CUDF_EXPECTS(
is_fixed_size_list(arrow_view), "Expected a fixed-size-list schema", cudf::data_type_error);
CUDF_EXPECTS(arrow_view->fixed_size >= 0,
"fixed-size-list width must be non-negative",
std::invalid_argument);
CUDF_EXPECTS(arrow_view->fixed_size <= std::numeric_limits<int32_t>::max(),
"fixed-size-list width exceeds the INT32 LIST offset range",
std::overflow_error);
return static_cast<int32_t>(arrow_view->fixed_size);
}

fixed_size_list_layout get_fixed_size_list_layout(ArrowSchemaView const* arrow_view,
ArrowArray const* input)
{
CUDF_EXPECTS(input->offset >= 0 && input->length >= 0,
"fixed-size-list offset and length must be non-negative",
std::invalid_argument);

constexpr auto max_column_size = static_cast<int64_t>(std::numeric_limits<size_type>::max());
constexpr auto max_row_count = max_column_size - 1;
CUDF_EXPECTS(input->length <= max_row_count,
"fixed-size-list length exceeds cuDF's maximum supported row count "
"(cudf::size_type)",
std::overflow_error);
CUDF_EXPECTS(input->offset <= std::numeric_limits<int64_t>::max() - input->length,
"fixed-size-list row bounds overflow Arrow's int64 representation",
std::overflow_error);

auto const width = fixed_size_list_width(arrow_view);
auto const num_rows = static_cast<size_type>(input->length);
auto const row_end = input->offset + input->length;

// Width zero is valid for a foreign Arrow producer even though nanoarrow's schema builder
// rejects it. Its offsets and child bounds are all zero.
if (width == 0) { return {width, num_rows, input->offset, row_end, 0, 0, 0}; }

CUDF_EXPECTS(row_end <= std::numeric_limits<int64_t>::max() / width,
"fixed-size-list child bounds overflow Arrow's int64 representation",
std::overflow_error);
auto const child_length = input->length * width;
CUDF_EXPECTS(child_length <= std::numeric_limits<int32_t>::max(),
"fixed-size-list child elements exceed the INT32 LIST offset range",
std::overflow_error);
CUDF_EXPECTS(child_length <= max_column_size,
"Number of fixed-size-list child elements exceeds cuDF's maximum supported "
"row count (cudf::size_type)",
std::overflow_error);

return {
width, num_rows, input->offset, row_end, input->offset * width, child_length, row_end * width};
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

ArrowType id_to_arrow_type(cudf::type_id id)
{
switch (id) {
Expand Down
51 changes: 50 additions & 1 deletion cpp/src/interop/arrow_utilities.hpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand All @@ -10,6 +10,8 @@

#include <nanoarrow/nanoarrow.h>

#include <cstdint>

namespace cudf {
namespace detail {

Expand All @@ -28,6 +30,53 @@ static constexpr int fixed_width_data_buffer_idx = 1;
*/
data_type arrow_to_cudf_type(ArrowSchemaView const* arrow_view);

/**
* @brief Check whether the given schema view describes an Arrow fixed-size-list
*
* @param arrow_view SchemaView to check
* @return True if the schema describes a fixed-size-list
*/
bool is_fixed_size_list(ArrowSchemaView const* arrow_view);

/**
* @brief Validated physical bounds for an Arrow fixed-size-list array
*/
struct fixed_size_list_layout {
int32_t width; ///< Child elements per row and LIST offset increment
Comment thread
wence- marked this conversation as resolved.
size_type num_rows; ///< Number of output rows
int64_t row_offset; ///< First logical row in the Arrow array
int64_t row_end; ///< One-past-last logical row
int64_t child_offset; ///< First referenced child element
int64_t child_length; ///< Number of referenced child elements
int64_t child_end; ///< One-past-last referenced child element
};

/**
* @brief Return the number of child elements per row of a fixed-size-list schema
*
* @throw cudf::data_type_error if `arrow_view` is not a fixed-size-list
* @throw std::invalid_argument if the declared width is negative
* @throw std::overflow_error if the declared width exceeds the INT32 LIST offset range
*
* @param arrow_view SchemaView to pull the fixed size from
* @return Number of child elements per row
*/
int32_t fixed_size_list_width(ArrowSchemaView const* arrow_view);

/**
* @brief Validate and compute fixed-size-list row and child bounds
*
* @throw std::invalid_argument if row metadata is negative
* @throw std::overflow_error if Arrow bounds overflow `int64_t`, output row counts exceed
* `size_type`, or synthesized LIST offsets exceed INT32
*
* @param arrow_view Fixed-size-list schema view
* @param input Arrow array carrying row offset and length
* @return Validated source bounds and output sizes
*/
fixed_size_list_layout get_fixed_size_list_layout(ArrowSchemaView const* arrow_view,
ArrowArray const* input);

/**
* @brief Map cudf column type id to ArrowType id
*
Expand Down
54 changes: 44 additions & 10 deletions cpp/src/interop/from_arrow_device.cu
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/

#include "arrow_utilities.hpp"
#include "from_arrow_host.hpp"

#include <cudf/column/column_view.hpp>
#include <cudf/copying.hpp>
Expand Down Expand Up @@ -321,15 +322,26 @@ dispatch_tuple_t dispatch_from_arrow_device::operator()<cudf::list_view>(
CUDF_EXPECTS(schema->type != NANOARROW_TYPE_LARGE_LIST,
"Large list types are not supported",
cudf::data_type_error);
size_type const num_rows = input->length;
size_type const offset = input->offset;
auto const fixed_size = is_fixed_size_list(schema);
auto const layout =
fixed_size ? get_fixed_size_list_layout(schema, input) : fixed_size_list_layout{};

if (fixed_size) {
constexpr auto max_row_count = static_cast<int64_t>(std::numeric_limits<size_type>::max()) - 1;
CUDF_EXPECTS(layout.row_end <= max_row_count,
"fixed-size-list device row bounds exceed cuDF's maximum supported row count "
"(cudf::size_type)",
std::overflow_error);
CUDF_EXPECTS(layout.child_end <= std::numeric_limits<int32_t>::max(),
"fixed-size-list device child bounds exceed the INT32 LIST offset range",
std::overflow_error);
CUDF_EXPECTS(input->children[0]->length >= layout.child_end,
"fixed-size-list child is shorter than its parent layout requires",
std::invalid_argument);
}
size_type const num_rows = fixed_size ? layout.num_rows : input->length;
size_type const offset = fixed_size ? static_cast<size_type>(layout.row_offset) : input->offset;
size_type const null_count = input->null_count;
auto offsets_view = column_view{data_type(type_id::INT32),
(num_rows == 0) ? 0 : (offset + num_rows + 1),
input->buffers[fixed_width_data_buffer_idx],
nullptr,
0,
0};

ArrowSchemaView child_schema_view;
NANOARROW_THROW_NOT_OK(
Expand All @@ -341,8 +353,30 @@ dispatch_tuple_t dispatch_from_arrow_device::operator()<cudf::list_view>(
// in the scenario where we were sliced and there are more elements in the child_view
// than can be referenced by the sliced offsets, we need to slice the child_view
// so that when `get_sliced_child` is called, we still produce the right result
auto max_child_offset =
num_rows == 0 ? 0 : cudf::detail::get_value<int32_t>(offsets_view, offset + num_rows, stream);
column_view offsets_view;
size_type max_child_offset = 0;
if (fixed_size) {
// fixed-size-list arrays carry no offsets buffer, so synthesize {0, w, 2w, ...}.
// these are absolute rather than normalized because the outer column_view applies a
// single offset to both the null mask and the children.
max_child_offset = (num_rows == 0) ? 0 : static_cast<size_type>(layout.child_end);
if (num_rows == 0) {
offsets_view = column_view{data_type{type_id::INT32}, 0, nullptr, nullptr, 0, 0};
} else {
owned.emplace_back(make_fixed_size_list_offsets(
static_cast<size_type>(layout.row_end) + 1, layout.width, stream, mr));
offsets_view = owned.back()->view();
Comment thread
0guban0v marked this conversation as resolved.
}
} else {
offsets_view = column_view{data_type(type_id::INT32),
(num_rows == 0) ? 0 : (offset + num_rows + 1),
input->buffers[fixed_width_data_buffer_idx],
nullptr,
0,
0};
max_child_offset =
num_rows == 0 ? 0 : cudf::detail::get_value<int32_t>(offsets_view, offset + num_rows, stream);
}
child_view = cudf::slice(child_view, {0, max_child_offset}, stream).front();

return std::make_tuple<column_view, owned_columns_t>(
Expand Down
Loading
Loading