Skip to content

test(parquet): replace InMemoryArrayReader with PrimitiveArrayReader in tests #9847

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alamb merged 1 commit into from
May 5, 2026
Merged

test(parquet): replace InMemoryArrayReader with PrimitiveArrayReader in tests #9847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 31 additions & 86 deletions parquet/src/arrow/array_reader/fixed_size_list_array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ mod tests {
use super::*;
use crate::arrow::{
ArrowWriter,
array_reader::{ListArrayReader, test_util::InMemoryArrayReader},
array_reader::{ListArrayReader, test_util::make_int32_page_reader},
arrow_reader::{ArrowReaderBuilder, ArrowReaderOptions, ParquetRecordBatchReader},
};
use arrow::datatypes::{Field, Int32Type};
Expand Down Expand Up @@ -254,28 +254,16 @@ mod tests {
3,
);

let array = Arc::new(PrimitiveArray::<Int32Type>::from(vec![
None,
Some(1),
None,
Some(2),
None,
Some(3),
Some(4),
Some(5),
None,
None,
None,
]));
let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
array,
Some(vec![0, 3, 2, 3, 0, 3, 3, 3, 2, 2, 2]),
Some(vec![0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1]),
let item_array_reader = make_int32_page_reader(
&[1, 2, 3, 4, 5],
&[0, 3, 2, 3, 0, 3, 3, 3, 2, 2, 2],
&[0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1],
3,
1,
);

let mut list_array_reader = FixedSizeListArrayReader::new(
Box::new(item_array_reader),
item_array_reader,
3,
ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 3),
2,
Expand Down Expand Up @@ -303,25 +291,16 @@ mod tests {
2,
);

let array = Arc::new(PrimitiveArray::<Int32Type>::from(vec![
Some(1),
None,
Some(2),
Some(3),
None,
None,
Some(4),
Some(5),
]));
let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
array,
Some(vec![2, 1, 2, 2, 1, 1, 2, 2]),
Some(vec![0, 1, 0, 1, 0, 1, 0, 1]),
let item_array_reader = make_int32_page_reader(
&[1, 2, 3, 4, 5],
&[2, 1, 2, 2, 1, 1, 2, 2],
&[0, 1, 0, 1, 0, 1, 0, 1],
2,
1,
);

let mut list_array_reader = FixedSizeListArrayReader::new(
Box::new(item_array_reader),
item_array_reader,
2,
ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 2),
1,
Expand Down Expand Up @@ -381,28 +360,15 @@ mod tests {

let expected = FixedSizeListArray::from(l1);

let values = Arc::new(PrimitiveArray::<Int32Type>::from(vec![
None,
Some(1),
Some(2),
None,
Some(3),
None,
Some(4),
Some(5),
None,
None,
]));

let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
values,
Some(vec![0, 5, 5, 4, 5, 0, 5, 5, 4, 4]),
Some(vec![0, 0, 2, 0, 2, 0, 0, 2, 0, 2]),
let item_array_reader = make_int32_page_reader(
&[1, 2, 3, 4, 5],
&[0, 5, 5, 4, 5, 0, 5, 5, 4, 4],
&[0, 0, 2, 0, 2, 0, 0, 2, 0, 2],
5,
2,
);

let l2 =
FixedSizeListArrayReader::new(Box::new(item_array_reader), 2, l2_type, 4, 2, false);
let l2 = FixedSizeListArrayReader::new(item_array_reader, 2, l2_type, 4, 2, false);
let mut l1 = FixedSizeListArrayReader::new(Box::new(l2), 1, l1_type, 3, 1, true);

let expected_1 = expected.slice(0, 2);
Expand All @@ -423,18 +389,10 @@ mod tests {
0,
);

let array = Arc::new(PrimitiveArray::<Int32Type>::from(vec![
None, None, None, None,
]));
let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
array,
Some(vec![0, 1, 0, 1]),
Some(vec![0, 0, 0, 0]),
);
let item_array_reader = make_int32_page_reader(&[], &[0, 1, 0, 1], &[0, 0, 0, 0], 2, 1);

let mut list_array_reader = FixedSizeListArrayReader::new(
Box::new(item_array_reader),
item_array_reader,
0,
ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 0),
2,
Expand Down Expand Up @@ -467,33 +425,20 @@ mod tests {
builder.append(false);
let expected = builder.finish();

let array = Arc::new(PrimitiveArray::<Int32Type>::from(vec![
Some(1),
None,
Some(3),
None,
Some(4),
None,
Some(5),
Some(6),
None,
None,
None,
]));

let inner_type = ArrowType::List(Arc::new(Field::new_list_field(ArrowType::Int32, true)));
let list_type =
ArrowType::FixedSizeList(Arc::new(Field::new_list_field(inner_type.clone(), true)), 2);

let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
array,
Some(vec![5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0]),
Some(vec![0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0]),
let item_array_reader = make_int32_page_reader(
&[1, 3, 4, 5, 6],
&[5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0],
&[0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0],
5,
2,
);

let inner_array_reader =
ListArrayReader::<i32>::new(Box::new(item_array_reader), inner_type, 4, 2, true);
ListArrayReader::<i32>::new(item_array_reader, inner_type, 4, 2, true);

let mut list_array_reader =
FixedSizeListArrayReader::new(Box::new(inner_array_reader), 2, list_type, 2, 1, true);
Expand Down
94 changes: 23 additions & 71 deletions parquet/src/arrow/array_reader/list_array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ mod tests {
use super::*;
use crate::arrow::array_reader::ArrayReaderBuilder;
use crate::arrow::array_reader::list_array::ListArrayReader;
use crate::arrow::array_reader::test_util::InMemoryArrayReader;
use crate::arrow::array_reader::test_util::make_int32_page_reader;
use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE;
use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics;
use crate::arrow::schema::parquet_to_arrow_schema_and_fields;
Expand All @@ -257,7 +257,7 @@ mod tests {
use crate::file::reader::{FileReader, SerializedFileReader};
use crate::schema::parser::parse_message_type;
use crate::schema::types::SchemaDescriptor;
use arrow::datatypes::{Field, Int32Type as ArrowInt32, Int32Type};
use arrow::datatypes::{Field, Int32Type};
use arrow_array::{Array, PrimitiveArray};
use arrow_data::ArrayDataBuilder;
use arrow_schema::Fields;
Expand Down Expand Up @@ -354,36 +354,15 @@ mod tests {

let expected = GenericListArray::<OffsetSize>::from(l1);

let values = Arc::new(PrimitiveArray::<Int32Type>::from(vec![
Some(1),
None,
None,
Some(4),
None,
None,
Some(7),
None,
Some(1),
Some(2),
Some(3),
Some(4),
None,
Some(6),
None,
None,
None,
Some(11),
]));

let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
values,
Some(vec![6, 5, 3, 6, 4, 2, 6, 4, 6, 6, 6, 6, 5, 6, 3, 0, 1, 6]),
Some(vec![0, 3, 2, 2, 2, 1, 1, 1, 1, 3, 3, 2, 3, 3, 2, 0, 0, 0]),
let item_array_reader = make_int32_page_reader(
&[1, 4, 7, 1, 2, 3, 4, 6, 11],
&[6, 5, 3, 6, 4, 2, 6, 4, 6, 6, 6, 6, 5, 6, 3, 0, 1, 6],
&[0, 3, 2, 2, 2, 1, 1, 1, 1, 3, 3, 2, 3, 3, 2, 0, 0, 0],
6,
3,
);

let l3 =
ListArrayReader::<OffsetSize>::new(Box::new(item_array_reader), l3_type, 5, 3, true);
let l3 = ListArrayReader::<OffsetSize>::new(item_array_reader, l3_type, 5, 3, true);

let l2 = ListArrayReader::<OffsetSize>::new(Box::new(l3), l2_type, 3, 2, false);

Expand Down Expand Up @@ -411,28 +390,16 @@ mod tests {
Some(vec![None, Some(1)]),
]);

let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
Some(1),
None,
Some(2),
None,
Some(3),
Some(4),
None,
None,
None,
Some(1),
]));

let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
array,
Some(vec![2, 1, 2, 0, 2, 2, 0, 0, 1, 2]),
Some(vec![0, 1, 1, 0, 0, 1, 0, 0, 0, 1]),
let item_array_reader = make_int32_page_reader(
&[1, 2, 3, 4, 1],
&[2, 1, 2, 0, 2, 2, 0, 0, 1, 2],
&[0, 1, 1, 0, 0, 1, 0, 0, 0, 1],
2,
1,
);

let mut list_array_reader = ListArrayReader::<OffsetSize>::new(
Box::new(item_array_reader),
item_array_reader,
list_type::<OffsetSize>(ArrowType::Int32, true),
1,
1,
Expand Down Expand Up @@ -460,31 +427,16 @@ mod tests {
Some(vec![None, Some(1)]),
]);

let array = Arc::new(PrimitiveArray::<ArrowInt32>::from(vec![
Some(1),
None,
Some(2),
None,
None,
Some(3),
Some(4),
None,
None,
None,
None,
None,
Some(1),
]));

let item_array_reader = InMemoryArrayReader::new(
ArrowType::Int32,
array,
Some(vec![3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3]),
Some(vec![0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1]),
let item_array_reader = make_int32_page_reader(
&[1, 2, 3, 4, 1],
&[3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3],
&[0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1],
3,
1,
);

let mut list_array_reader = ListArrayReader::<OffsetSize>::new(
Box::new(item_array_reader),
item_array_reader,
list_type::<OffsetSize>(ArrowType::Int32, true),
2,
1,
Expand Down
Loading
Loading