From 312e256b342ba93e18b827d5609c1bdc37ef384f Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Wed, 9 Sep 2026 15:21:29 +0800 Subject: [PATCH 1/2] docs: point ParquetFormat setter/getter docs to the ParquetOptions definitions --- datafusion/common/src/config.rs | 8 +++ .../datasource-parquet/src/file_format.rs | 66 +++++++++---------- docs/source/user-guide/configs.md | 4 +- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index f770cebf88f95..03888be979edb 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1348,6 +1348,10 @@ config_namespace! { /// (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, /// and `Binary/BinaryLarge` with `BinaryView`. + /// + /// The parquet reader is optimized for reading `Utf8View` and `BinaryView`, + /// so such queries are significantly faster than reading `Utf8`/`Binary` + /// and then casting to the view types. pub schema_force_view_types: bool, default = true /// (reading) If true, parquet reader will read columns of @@ -1356,6 +1360,10 @@ config_namespace! { /// Parquet files generated by some legacy writers do not correctly set /// the UTF8 flag for strings, causing string columns to be loaded as /// BLOB instead. + /// + /// The parquet reader has special optimizations for `Utf8` validation, + /// so reading such columns as strings is significantly faster than + /// reading them as binary and then casting to string. pub binary_as_string: bool, default = false /// (reading) If true, parquet reader will read columns of diff --git a/datafusion/datasource-parquet/src/file_format.rs b/datafusion/datasource-parquet/src/file_format.rs index 1b25a2c632510..fd608b27580ff 100644 --- a/datafusion/datasource-parquet/src/file_format.rs +++ b/datafusion/datasource-parquet/src/file_format.rs @@ -155,46 +155,47 @@ impl ParquetFormat { Self::default() } - /// Activate statistics based row group level pruning - /// - If `None`, defaults to value on `config_options` + /// Set [`pruning`] + /// + /// [`pruning`]: datafusion_common::config::ParquetOptions::pruning pub fn with_enable_pruning(mut self, enable: bool) -> Self { self.options.global.pruning = enable; self } - /// Return `true` if pruning is enabled + /// Get [`pruning`] + /// + /// [`pruning`]: datafusion_common::config::ParquetOptions::pruning pub fn enable_pruning(&self) -> bool { self.options.global.pruning } - /// Provide a hint to the size of the file metadata. If a hint is provided - /// the reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. - /// Without a hint, two read are required. One read to fetch the 8-byte parquet footer and then - /// another read to fetch the metadata length encoded in the footer. + /// Set [`metadata_size_hint`] /// - /// - If `None`, defaults to value on `config_options` + /// [`metadata_size_hint`]: datafusion_common::config::ParquetOptions::metadata_size_hint pub fn with_metadata_size_hint(mut self, size_hint: Option) -> Self { self.options.global.metadata_size_hint = size_hint; self } - /// Return the metadata size hint if set + /// Get [`metadata_size_hint`] + /// + /// [`metadata_size_hint`]: datafusion_common::config::ParquetOptions::metadata_size_hint pub fn metadata_size_hint(&self) -> Option { self.options.global.metadata_size_hint } - /// Tell the parquet reader to skip any metadata that may be in - /// the file Schema. This can help avoid schema conflicts due to - /// metadata. + /// Set [`skip_metadata`] /// - /// - If `None`, defaults to value on `config_options` + /// [`skip_metadata`]: datafusion_common::config::ParquetOptions::skip_metadata pub fn with_skip_metadata(mut self, skip_metadata: bool) -> Self { self.options.global.skip_metadata = skip_metadata; self } - /// Returns `true` if schema metadata will be cleared prior to - /// schema merging. + /// Get [`skip_metadata`] + /// + /// [`skip_metadata`]: datafusion_common::config::ParquetOptions::skip_metadata pub fn skip_metadata(&self) -> bool { self.options.global.skip_metadata } @@ -210,49 +211,46 @@ impl ParquetFormat { &self.options } - /// Return `true` if should use view types. - /// - /// If this returns true, DataFusion will instruct the parquet reader - /// to read string / binary columns using view `StringView` or `BinaryView` - /// if the table schema specifies those types, regardless of any embedded metadata - /// that may specify an alternate Arrow type. The parquet reader is optimized - /// for reading `StringView` and `BinaryView` and such queries are significantly faster. + /// Get [`schema_force_view_types`] /// - /// If this returns false, the parquet reader will read the columns according to the - /// defaults or any embedded Arrow type information. This may result in reading - /// `StringArrays` and then casting to `StringViewArray` which is less efficient. + /// [`schema_force_view_types`]: datafusion_common::config::ParquetOptions::schema_force_view_types pub fn force_view_types(&self) -> bool { self.options.global.schema_force_view_types } - /// If true, will use view types. See [`Self::force_view_types`] for details + /// Set [`schema_force_view_types`] + /// + /// [`schema_force_view_types`]: datafusion_common::config::ParquetOptions::schema_force_view_types pub fn with_force_view_types(mut self, use_views: bool) -> Self { self.options.global.schema_force_view_types = use_views; self } - /// Return `true` if binary types will be read as strings. + /// Get [`binary_as_string`] /// - /// If this returns true, DataFusion will instruct the parquet reader - /// to read binary columns such as `Binary` or `BinaryView` as the - /// corresponding string type such as `Utf8` or `LargeUtf8`. - /// The parquet reader has special optimizations for `Utf8` and `LargeUtf8` - /// validation, and such queries are significantly faster than reading - /// binary columns and then casting to string columns. + /// [`binary_as_string`]: datafusion_common::config::ParquetOptions::binary_as_string pub fn binary_as_string(&self) -> bool { self.options.global.binary_as_string } - /// If true, will read binary types as strings. See [`Self::binary_as_string`] for details + /// Set [`binary_as_string`] + /// + /// [`binary_as_string`]: datafusion_common::config::ParquetOptions::binary_as_string pub fn with_binary_as_string(mut self, binary_as_string: bool) -> Self { self.options.global.binary_as_string = binary_as_string; self } + /// Get [`coerce_int96`] + /// + /// [`coerce_int96`]: datafusion_common::config::ParquetOptions::coerce_int96 pub fn coerce_int96(&self) -> Option { self.options.global.coerce_int96.clone() } + /// Set [`coerce_int96`] + /// + /// [`coerce_int96`]: datafusion_common::config::ParquetOptions::coerce_int96 pub fn with_coerce_int96(mut self, time_unit: Option) -> Self { self.options.global.coerce_int96 = time_unit; self diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index fbeb38f1cebe9..1f90247a731af 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -87,8 +87,8 @@ The following configuration settings are available: | datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | | datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | | datafusion.execution.parquet.force_filter_selections | false | (reading) Force the use of RowSelections for filter results, when pushdown_filters is enabled. If false, the reader will automatically choose between a RowSelection and a Bitmap based on the number and pattern of selected rows. | -| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. | -| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. | +| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. | +| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. | | datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | | datafusion.execution.parquet.coerce_int96_tz | NULL | (reading) Optional timezone applied to INT96 columns when `coerce_int96` is set. When `Some`, INT96 columns coerce to `Timestamp(, Some())` instead of the default `Timestamp(, None)`. Spark and other systems write INT96 values as UTC-adjusted instants, so callers that need the resulting Arrow type to be timezone-aware (e.g. for Spark `TimestampType` semantics) should set this to `"UTC"`. No effect when `coerce_int96` is `None`. | | datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | From a1d514b3138b45520aa4fc0f01dd6b8e151c0031 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Wed, 9 Sep 2026 16:04:24 +0800 Subject: [PATCH 2/2] fix test --- datafusion/sqllogictest/test_files/information_schema.slt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 4551f21f21645..014d8a2d169fb 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -394,7 +394,7 @@ datafusion.execution.meta_fetch_concurrency 32 Number of files to read in parall datafusion.execution.minimum_parallel_output_files 4 Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. datafusion.execution.objectstore_writer_buffer_size 10485760 Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. datafusion.execution.parquet.allow_single_file_parallelism true (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. -datafusion.execution.parquet.binary_as_string false (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. +datafusion.execution.parquet.binary_as_string false (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. The parquet reader has special optimizations for `Utf8` validation, so reading such columns as strings is significantly faster than reading them as binary and then casting to string. datafusion.execution.parquet.bloom_filter_fpp NULL (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting datafusion.execution.parquet.bloom_filter_ndv NULL (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting datafusion.execution.parquet.bloom_filter_on_read true (reading) Use any available bloom filters when reading parquet files @@ -425,7 +425,7 @@ datafusion.execution.parquet.metadata_size_hint 524288 (reading) If specified, t datafusion.execution.parquet.pruning true (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file datafusion.execution.parquet.pushdown_filters false (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". datafusion.execution.parquet.reorder_filters false (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query -datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. +datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. The parquet reader is optimized for reading `Utf8View` and `BinaryView`, so such queries are significantly faster than reading `Utf8`/`Binary` and then casting to the view types. datafusion.execution.parquet.skip_arrow_metadata false (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata datafusion.execution.parquet.statistics_enabled page (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting