From f747b527e2b9288ac091be758a6fe6505448c732 Mon Sep 17 00:00:00 2001 From: Zehua Zou Date: Tue, 1 Sep 2026 14:31:21 +0800 Subject: [PATCH 1/5] allow CSV reader to ignore extra columns --- cpp/src/arrow/csv/options.h | 2 + cpp/src/arrow/csv/parser.cc | 62 +++++++++++++++++++++++-------- cpp/src/arrow/csv/parser_test.cc | 13 +++++++ cpp/src/arrow/csv/reader_test.cc | 21 +++++++++++ cpp/src/arrow/dataset/file_csv.cc | 10 +++-- 5 files changed, 88 insertions(+), 20 deletions(-) diff --git a/cpp/src/arrow/csv/options.h b/cpp/src/arrow/csv/options.h index 5d83f9cb491b..41d0b63dacbb 100644 --- a/cpp/src/arrow/csv/options.h +++ b/cpp/src/arrow/csv/options.h @@ -63,6 +63,8 @@ struct ARROW_EXPORT ParseOptions { InvalidRowHandler invalid_row_handler; /// Whether rows with fewer columns than expected are padded with nulls. bool pad_short_rows = false; + /// Whether rows with more columns than expected should ignore the extra columns. + bool ignore_extra_columns = false; /// Create parsing options with default values static ParseOptions Defaults(); diff --git a/cpp/src/arrow/csv/parser.cc b/cpp/src/arrow/csv/parser.cc index 6a6138cec856..4dc3415875f5 100644 --- a/cpp/src/arrow/csv/parser.cc +++ b/cpp/src/arrow/csv/parser.cc @@ -287,7 +287,23 @@ class BlockParserImpl { DCHECK_GT(data_end, data); - auto FinishField = [&]() { values_writer->FinishField(parsed_writer); }; + bool ignoring_extra_field = false; + + auto IsExtraField = [&]() { + return batch_.num_cols_ >= 0 && options_.ignore_extra_columns && + num_cols >= batch_.num_cols_; + }; + auto StartField = [&](bool quoted) { + ignoring_extra_field = IsExtraField(); + if (!ignoring_extra_field) { + values_writer->StartField(quoted); + } + }; + auto FinishField = [&]() { + if (!IsExtraField()) { + values_writer->FinishField(parsed_writer); + } + }; values_writer->BeginLine(); parsed_writer->BeginLine(); @@ -314,7 +330,7 @@ class BlockParserImpl { // At the start of a field if (*data == options_.delimiter) { // Empty cells are very common in some files, shortcut them - values_writer->StartField(false /* quoted */); + StartField(false /* quoted */); FinishField(); ++data; ++num_cols; @@ -328,17 +344,18 @@ class BlockParserImpl { if (SpecializedOptions::quoting && ARROW_PREDICT_FALSE(*data == options_.quote_char)) { ++data; - values_writer->StartField(true /* quoted */); + StartField(true /* quoted */); goto InQuotedField; } else { - values_writer->StartField(false /* quoted */); + StartField(false /* quoted */); goto InField; } InField: // Inside a non-quoted part of a field if (UseBulkFilter) { - const char* bulk_end = RunBulkFilter(parsed_writer, data, data_end, bulk_filter); + const char* bulk_end = + RunBulkFilter(parsed_writer, data, data_end, bulk_filter, ignoring_extra_field); if (ARROW_PREDICT_FALSE(bulk_end == nullptr)) { if (is_final) { data = data_end; @@ -358,7 +375,9 @@ class BlockParserImpl { goto AbortLine; } c = *data++; - parsed_writer->PushFieldChar(c); + if (!ignoring_extra_field) { + parsed_writer->PushFieldChar(c); + } goto InField; } if (ARROW_PREDICT_FALSE(c == options_.delimiter)) { @@ -376,13 +395,16 @@ class BlockParserImpl { goto LineEnd; } } - parsed_writer->PushFieldChar(c); + if (!ignoring_extra_field) { + parsed_writer->PushFieldChar(c); + } goto InField; InQuotedField: // Inside a quoted part of a field if (UseBulkFilter) { - const char* bulk_end = RunBulkFilter(parsed_writer, data, data_end, bulk_filter); + const char* bulk_end = + RunBulkFilter(parsed_writer, data, data_end, bulk_filter, ignoring_extra_field); if (ARROW_PREDICT_FALSE(bulk_end == nullptr)) { if (is_final) { data = data_end; @@ -401,7 +423,9 @@ class BlockParserImpl { goto AbortLine; } c = *data++; - parsed_writer->PushFieldChar(c); + if (!ignoring_extra_field) { + parsed_writer->PushFieldChar(c); + } goto InQuotedField; } if (ARROW_PREDICT_FALSE(c == options_.quote_char)) { @@ -414,7 +438,9 @@ class BlockParserImpl { goto InField; } } - parsed_writer->PushFieldChar(c); + if (!ignoring_extra_field) { + parsed_writer->PushFieldChar(c); + } goto InQuotedField; FieldEnd: @@ -436,11 +462,11 @@ class BlockParserImpl { } else if (options_.pad_short_rows && num_cols < batch_.num_cols_) { batch_.missing_fields_.push_back({batch_.num_rows_, num_cols}); while (num_cols < batch_.num_cols_) { - values_writer->StartField(false /* quoted */); + StartField(false /* quoted */); FinishField(); ++num_cols; } - } else { + } else if (!options_.ignore_extra_columns || num_cols < batch_.num_cols_) { return HandleInvalidRow(values_writer, parsed_writer, start, data, num_cols, out_data); } @@ -466,9 +492,10 @@ class BlockParserImpl { batch_.num_cols_ = 1; } // Record as row of empty (null?) values - while (num_cols++ < batch_.num_cols_) { - values_writer->StartField(false /* quoted */); + while (num_cols < batch_.num_cols_) { + StartField(false /* quoted */); FinishField(); + ++num_cols; } ++batch_.num_rows_; } @@ -479,7 +506,8 @@ class BlockParserImpl { template const char* RunBulkFilter(DataWriter* data_writer, const char* data, const char* data_end, - const SpecializedBulkFilter& bulk_filter) { + const SpecializedBulkFilter& bulk_filter, + bool ignoring_extra_field) { while (true) { using WordType = typename SpecializedBulkFilter::WordType; @@ -495,7 +523,9 @@ class BlockParserImpl { return data; } // No special chars - data_writer->PushFieldWord(word); + if (!ignoring_extra_field) { + data_writer->PushFieldWord(word); + } data += sizeof(WordType); } } diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index c03f492ed278..a15e8ed04e3f 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -298,6 +298,19 @@ TEST(BlockParser, PadShortRows) { ASSERT_EQ(last_row_missing, std::vector({false, false, true})); } +TEST(BlockParser, IgnoreExtraColumns) { + auto options = ParseOptions::Defaults(); + options.ignore_extra_columns = true; + + BlockParser parser(options, /*num_cols=*/2); + AssertParseOk(parser, "a,\"b\",c,\nd,e\n"); + AssertColumnsEq(parser, {{"a", "d"}, {"b", "e"}}, {{false, false}, {true, false}}); + + BlockParser final_parser(options, /*num_cols=*/2); + AssertParseFinal(final_parser, "a,b,"); + AssertColumnsEq(final_parser, {{"a"}, {"b"}}); +} + TEST(BlockParser, EmptyHeader) { // Cannot infer number of columns uint32_t out_size; diff --git a/cpp/src/arrow/csv/reader_test.cc b/cpp/src/arrow/csv/reader_test.cc index 2493cb66271e..009bbd5fc25a 100644 --- a/cpp/src/arrow/csv/reader_test.cc +++ b/cpp/src/arrow/csv/reader_test.cc @@ -644,6 +644,27 @@ TEST(ReaderTests, ShortRows) { ASSERT_TRUE(table->Equals(*expected_table)); } +TEST(ReaderTests, IgnoreExtraColumns) { + auto input = + std::make_shared(std::make_shared("a,b\n1,2,3\n4,5\n")); + auto parse_options = ParseOptions::Defaults(); + parse_options.ignore_extra_columns = true; + auto convert_options = ConvertOptions::Defaults(); + convert_options.default_column_type = int64(); + + ASSERT_OK_AND_ASSIGN(auto reader, TableReader::Make(io::default_io_context(), input, + ReadOptions::Defaults(), + parse_options, convert_options)); + ASSERT_OK_AND_ASSIGN(auto table, reader->Read()); + + auto expected_schema = schema({field("a", int64()), field("b", int64())}); + auto expected_table = TableFromJSON(expected_schema, {R"([ + {"a":1, "b":2}, + {"a":4, "b":5} + ])"}); + ASSERT_TRUE(table->Equals(*expected_table)); +} + TEST(ReaderTests, ShortRowsTypedConverters) { auto input = std::make_shared(std::make_shared("1,10\n2\n")); auto read_options = ReadOptions::Defaults(); diff --git a/cpp/src/arrow/dataset/file_csv.cc b/cpp/src/arrow/dataset/file_csv.cc index 079103fa7918..3b9e8d6ca204 100644 --- a/cpp/src/arrow/dataset/file_csv.cc +++ b/cpp/src/arrow/dataset/file_csv.cc @@ -164,11 +164,12 @@ Result> GetOrderedColumnNames( int32_t max_num_rows = read_options.skip_rows + 1; std::optional inspection_parse_options; const auto* parser_options = &parse_options; - if (parse_options.pad_short_rows) { - // Do not pad short rows while determining column names, since padding cannot - // synthesize missing names. Copy the parse options only when needed. + if (parse_options.pad_short_rows || parse_options.ignore_extra_columns) { + // Do not adjust row widths while determining column names: padding cannot + // synthesize missing names, and ignoring extra columns may discard columns. inspection_parse_options.emplace(parse_options); inspection_parse_options->pad_short_rows = false; + inspection_parse_options->ignore_extra_columns = false; parser_options = &*inspection_parse_options; } csv::BlockParser parser(pool, *parser_options, /*num_cols=*/-1, /*first_row=*/1, @@ -379,7 +380,8 @@ bool CsvFileFormat::Equals(const FileFormat& format) const { parse_options.escape_char == other_parse_options.escape_char && parse_options.newlines_in_values == other_parse_options.newlines_in_values && parse_options.ignore_empty_lines == other_parse_options.ignore_empty_lines && - parse_options.pad_short_rows == other_parse_options.pad_short_rows; + parse_options.pad_short_rows == other_parse_options.pad_short_rows && + parse_options.ignore_extra_columns == other_parse_options.ignore_extra_columns; } Result CsvFileFormat::IsSupported(const FileSource& source) const { From c220f52472dda90445ab38edeac2a669b28bff0c Mon Sep 17 00:00:00 2001 From: Zehua Zou Date: Fri, 4 Sep 2026 16:03:19 +0800 Subject: [PATCH 2/5] minor performance optimizations --- cpp/src/arrow/csv/lexing_internal.h | 19 +++++ cpp/src/arrow/csv/parser.cc | 108 +++++++++++++++------------- 2 files changed, 78 insertions(+), 49 deletions(-) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index b45af7a370d4..4618beae7b6f 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -35,6 +35,25 @@ class SpecializedOptions { static constexpr bool escaping = Escaping; }; +template + requires std::invocable...> +decltype(auto) DispatchBool(Fn&& fn, Rest... rest) { + if constexpr (sizeof...(Rest) == 0) { + return std::forward(fn)(CompiledBools{}...); + } else { + return [&](bool head, auto... tail) -> decltype(auto) { + if (head) { + return DispatchBool(std::forward(fn), + tail...); + } else { + return DispatchBool(std::forward(fn), + tail...); + } + }(rest...); + } +} + // // Bulk filters for packed character matching. // These filters allow checking multiple CSV bytes at once for specific diff --git a/cpp/src/arrow/csv/parser.cc b/cpp/src/arrow/csv/parser.cc index 4dc3415875f5..acdadda70ce9 100644 --- a/cpp/src/arrow/csv/parser.cc +++ b/cpp/src/arrow/csv/parser.cc @@ -18,8 +18,10 @@ #include "arrow/csv/parser.h" #include +#include #include #include +#include #include #include "arrow/csv/lexing_internal.h" @@ -59,6 +61,14 @@ Status MismatchingColumns(const InvalidRow& row) { inline bool IsControlChar(uint8_t c) { return c < ' '; } +template +constexpr bool ShouldWrite(bool ignoring_extra_field) { + if constexpr (IgnoreExtraColumns) { + return !ignoring_extra_field; + } + return true; +} + // A helper class allocating the buffer for parsed values and writing into it // without any further resizes, except at the end. class PresizedDataWriter { @@ -276,8 +286,8 @@ class BlockParserImpl { return MismatchingColumns(row); } - template + template Status ParseLine(ValueDescWriter* values_writer, DataWriter* parsed_writer, const char* data, const char* data_end, bool is_final, const char** out_data, const BulkFilter& bulk_filter) { @@ -290,17 +300,22 @@ class BlockParserImpl { bool ignoring_extra_field = false; auto IsExtraField = [&]() { - return batch_.num_cols_ >= 0 && options_.ignore_extra_columns && - num_cols >= batch_.num_cols_; + if constexpr (!IgnoreExtraColumns) { + return false; + } + return batch_.num_cols_ >= 0 && num_cols >= batch_.num_cols_; }; auto StartField = [&](bool quoted) { - ignoring_extra_field = IsExtraField(); - if (!ignoring_extra_field) { - values_writer->StartField(quoted); + if (ARROW_PREDICT_TRUE(ShouldWrite(ignoring_extra_field))) { + if (ARROW_PREDICT_FALSE(IsExtraField())) { + ignoring_extra_field = true; + } else { + values_writer->StartField(quoted); + } } }; auto FinishField = [&]() { - if (!IsExtraField()) { + if (ARROW_PREDICT_TRUE(ShouldWrite(ignoring_extra_field))) { values_writer->FinishField(parsed_writer); } }; @@ -354,8 +369,8 @@ class BlockParserImpl { InField: // Inside a non-quoted part of a field if (UseBulkFilter) { - const char* bulk_end = - RunBulkFilter(parsed_writer, data, data_end, bulk_filter, ignoring_extra_field); + const char* bulk_end = RunBulkFilter( + parsed_writer, data, data_end, bulk_filter, ignoring_extra_field); if (ARROW_PREDICT_FALSE(bulk_end == nullptr)) { if (is_final) { data = data_end; @@ -375,7 +390,7 @@ class BlockParserImpl { goto AbortLine; } c = *data++; - if (!ignoring_extra_field) { + if (ARROW_PREDICT_TRUE(ShouldWrite(ignoring_extra_field))) { parsed_writer->PushFieldChar(c); } goto InField; @@ -395,7 +410,7 @@ class BlockParserImpl { goto LineEnd; } } - if (!ignoring_extra_field) { + if (ARROW_PREDICT_TRUE(ShouldWrite(ignoring_extra_field))) { parsed_writer->PushFieldChar(c); } goto InField; @@ -403,8 +418,8 @@ class BlockParserImpl { InQuotedField: // Inside a quoted part of a field if (UseBulkFilter) { - const char* bulk_end = - RunBulkFilter(parsed_writer, data, data_end, bulk_filter, ignoring_extra_field); + const char* bulk_end = RunBulkFilter( + parsed_writer, data, data_end, bulk_filter, ignoring_extra_field); if (ARROW_PREDICT_FALSE(bulk_end == nullptr)) { if (is_final) { data = data_end; @@ -423,7 +438,7 @@ class BlockParserImpl { goto AbortLine; } c = *data++; - if (!ignoring_extra_field) { + if (ARROW_PREDICT_TRUE(ShouldWrite(ignoring_extra_field))) { parsed_writer->PushFieldChar(c); } goto InQuotedField; @@ -438,7 +453,7 @@ class BlockParserImpl { goto InField; } } - if (!ignoring_extra_field) { + if (ARROW_PREDICT_TRUE(ShouldWrite(ignoring_extra_field))) { parsed_writer->PushFieldChar(c); } goto InQuotedField; @@ -466,7 +481,7 @@ class BlockParserImpl { FinishField(); ++num_cols; } - } else if (!options_.ignore_extra_columns || num_cols < batch_.num_cols_) { + } else if (!IgnoreExtraColumns || num_cols < batch_.num_cols_) { return HandleInvalidRow(values_writer, parsed_writer, start, data, num_cols, out_data); } @@ -478,6 +493,10 @@ class BlockParserImpl { AbortLine: // Not a full line except perhaps if in final block if (is_final) { + if constexpr (IgnoreExtraColumns) { + // Handle an implicit trailing empty field after a delimiter. + ignoring_extra_field = IsExtraField(); + } goto LineEnd; } // Truncated line at end of block, rewind parsed state @@ -503,7 +522,7 @@ class BlockParserImpl { return Status::OK(); } - template + template const char* RunBulkFilter(DataWriter* data_writer, const char* data, const char* data_end, const SpecializedBulkFilter& bulk_filter, @@ -523,15 +542,15 @@ class BlockParserImpl { return data; } // No special chars - if (!ignoring_extra_field) { + if (ARROW_PREDICT_TRUE(ShouldWrite(ignoring_extra_field))) { data_writer->PushFieldWord(word); } data += sizeof(WordType); } } - template + template Status ParseChunk(ValueDescWriter* values_writer, DataWriter* parsed_writer, const char* data, const char* data_end, bool is_final, int32_t rows_in_chunk, const char** out_data, bool* finished_parsing, @@ -542,9 +561,9 @@ class BlockParserImpl { if (use_bulk_filter_) { while (data < data_end && batch_.num_rows_ < num_rows_deadline) { const char* line_end = data; - RETURN_NOT_OK((ParseLine(values_writer, parsed_writer, - data, data_end, is_final, - &line_end, bulk_filter))); + RETURN_NOT_OK((ParseLine( + values_writer, parsed_writer, data, data_end, is_final, &line_end, + bulk_filter))); RETURN_NOT_OK(values_writer->status()); if (line_end == data) { // Cannot parse any further @@ -556,9 +575,9 @@ class BlockParserImpl { } else { while (data < data_end && batch_.num_rows_ < num_rows_deadline) { const char* line_end = data; - RETURN_NOT_OK((ParseLine(values_writer, parsed_writer, - data, data_end, is_final, - &line_end, bulk_filter))); + RETURN_NOT_OK((ParseLine( + values_writer, parsed_writer, data, data_end, is_final, &line_end, + bulk_filter))); RETURN_NOT_OK(values_writer->status()); if (line_end == data) { // Cannot parse any further @@ -589,7 +608,7 @@ class BlockParserImpl { return Status::OK(); } - template + template Status ParseSpecialized(const std::vector& views, bool is_final, uint32_t* out_size) { internal::PreferredBulkFilterType bulk_filter(options_); @@ -628,9 +647,9 @@ class BlockParserImpl { ARROW_ASSIGN_OR_RAISE(auto values_writer, ResizableValueDescWriter::Make(pool_)); values_writer.Start(parsed_writer); - RETURN_NOT_OK(ParseChunk( + RETURN_NOT_OK((ParseChunk( &values_writer, &parsed_writer, data, data_end, is_final, rows_in_chunk, - &data, &finished_parsing, bulk_filter)); + &data, &finished_parsing, bulk_filter))); if (batch_.num_cols_ == -1) { return ParseError("Empty CSV file or block: cannot infer number of columns"); } @@ -666,9 +685,9 @@ class BlockParserImpl { PresizedValueDescWriter::Make(pool_, rows_in_chunk, batch_.num_cols_)); values_writer.Start(parsed_writer); - RETURN_NOT_OK(ParseChunk( + RETURN_NOT_OK((ParseChunk( &values_writer, &parsed_writer, data, data_end, is_final, rows_in_chunk, - &data, &finished_parsing, bulk_filter)); + &data, &finished_parsing, bulk_filter))); } DCHECK_GE(data, view.data()); DCHECK_LE(data, data_end); @@ -709,23 +728,14 @@ class BlockParserImpl { Status Parse(const std::vector& data, bool is_final, uint32_t* out_size) { - if (options_.quoting) { - if (options_.escaping) { - return ParseSpecialized>(data, is_final, - out_size); - } else { - return ParseSpecialized>(data, is_final, - out_size); - } - } else { - if (options_.escaping) { - return ParseSpecialized>(data, is_final, - out_size); - } else { - return ParseSpecialized>( - data, is_final, out_size); - } - } + return internal::DispatchBool( + [&](auto quoting, auto escaping, auto ignore_extra_columns) { + using SpecializedOptions = + internal::SpecializedOptions; + return ParseSpecialized( + data, is_final, out_size); + }, + options_.quoting, options_.escaping, options_.ignore_extra_columns); } protected: From 3cfdb0ef3d1c3c94d161381f1898ed84b1d4934b Mon Sep 17 00:00:00 2001 From: Zehua Zou Date: Tue, 8 Sep 2026 13:49:08 +0800 Subject: [PATCH 3/5] minor file changes --- cpp/src/arrow/csv/lexing_internal.h | 2 ++ cpp/src/arrow/csv/parser.cc | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index 4618beae7b6f..cc8ad9d2a602 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -17,9 +17,11 @@ #pragma once +#include #include #include #include +#include #include "arrow/csv/options.h" #include "arrow/util/simd.h" diff --git a/cpp/src/arrow/csv/parser.cc b/cpp/src/arrow/csv/parser.cc index acdadda70ce9..13c1a3cc1cb9 100644 --- a/cpp/src/arrow/csv/parser.cc +++ b/cpp/src/arrow/csv/parser.cc @@ -18,10 +18,8 @@ #include "arrow/csv/parser.h" #include -#include #include #include -#include #include #include "arrow/csv/lexing_internal.h" @@ -62,7 +60,7 @@ Status MismatchingColumns(const InvalidRow& row) { inline bool IsControlChar(uint8_t c) { return c < ' '; } template -constexpr bool ShouldWrite(bool ignoring_extra_field) { +constexpr bool ShouldWrite([[maybe_unused]] bool ignoring_extra_field) { if constexpr (IgnoreExtraColumns) { return !ignoring_extra_field; } From 83c0bb96efe12c217916112b7fafa6725b6e490a Mon Sep 17 00:00:00 2001 From: Zehua Zou Date: Tue, 8 Sep 2026 15:57:10 +0800 Subject: [PATCH 4/5] minor refactor --- cpp/src/arrow/csv/lexing_internal.h | 21 +++++++++++---------- cpp/src/arrow/csv/parser.cc | 9 ++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index cc8ad9d2a602..96135a71c30e 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -17,11 +17,11 @@ #pragma once -#include #include #include #include #include +#include #include "arrow/csv/options.h" #include "arrow/util/simd.h" @@ -37,20 +37,21 @@ class SpecializedOptions { static constexpr bool escaping = Escaping; }; -template - requires std::invocable...> -decltype(auto) DispatchBool(Fn&& fn, Rest... rest) { +template +decltype(auto) DispatchBool(Fn&& fn, Rest... rest) + requires requires { + std::forward(fn) + .template operator()...>(); + } +{ if constexpr (sizeof...(Rest) == 0) { - return std::forward(fn)(CompiledBools{}...); + return std::forward(fn).template operator()(); } else { return [&](bool head, auto... tail) -> decltype(auto) { if (head) { - return DispatchBool(std::forward(fn), - tail...); + return DispatchBool(std::forward(fn), tail...); } else { - return DispatchBool(std::forward(fn), - tail...); + return DispatchBool(std::forward(fn), tail...); } }(rest...); } diff --git a/cpp/src/arrow/csv/parser.cc b/cpp/src/arrow/csv/parser.cc index 13c1a3cc1cb9..20f984ffeb66 100644 --- a/cpp/src/arrow/csv/parser.cc +++ b/cpp/src/arrow/csv/parser.cc @@ -727,11 +727,10 @@ class BlockParserImpl { Status Parse(const std::vector& data, bool is_final, uint32_t* out_size) { return internal::DispatchBool( - [&](auto quoting, auto escaping, auto ignore_extra_columns) { - using SpecializedOptions = - internal::SpecializedOptions; - return ParseSpecialized( - data, is_final, out_size); + [&]() { + using SpecializedOptions = internal::SpecializedOptions; + return ParseSpecialized(data, is_final, + out_size); }, options_.quoting, options_.escaping, options_.ignore_extra_columns); } From 3000cb48568d1c00fc91f762aacc71c23972fd6e Mon Sep 17 00:00:00 2001 From: Zehua Zou Date: Wed, 9 Sep 2026 23:23:37 +0800 Subject: [PATCH 5/5] address review --- cpp/src/arrow/csv/lexing_internal.h | 4 ++++ cpp/src/arrow/csv/parser_test.cc | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/cpp/src/arrow/csv/lexing_internal.h b/cpp/src/arrow/csv/lexing_internal.h index 96135a71c30e..d5ca120eb58a 100644 --- a/cpp/src/arrow/csv/lexing_internal.h +++ b/cpp/src/arrow/csv/lexing_internal.h @@ -37,6 +37,7 @@ class SpecializedOptions { static constexpr bool escaping = Escaping; }; +/// Convert runtime boolean options into template arguments for a callable. template decltype(auto) DispatchBool(Fn&& fn, Rest... rest) requires requires { @@ -45,8 +46,11 @@ decltype(auto) DispatchBool(Fn&& fn, Rest... rest) } { if constexpr (sizeof...(Rest) == 0) { + // All runtime booleans have been appended to the compile-time pack. return std::forward(fn).template operator()(); } else { + // Split off the next runtime boolean, append its value to the compile-time pack, + // and recursively dispatch the remaining booleans. return [&](bool head, auto... tail) -> decltype(auto) { if (head) { return DispatchBool(std::forward(fn), tail...); diff --git a/cpp/src/arrow/csv/parser_test.cc b/cpp/src/arrow/csv/parser_test.cc index a15e8ed04e3f..bd09f9911831 100644 --- a/cpp/src/arrow/csv/parser_test.cc +++ b/cpp/src/arrow/csv/parser_test.cc @@ -311,6 +311,26 @@ TEST(BlockParser, IgnoreExtraColumns) { AssertColumnsEq(final_parser, {{"a"}, {"b"}}); } +TEST(BlockParser, PadAndIgnore) { + auto options = ParseOptions::Defaults(); + options.pad_short_rows = true; + options.ignore_extra_columns = true; + + BlockParser parser(options, /*num_cols=*/2); + AssertParseFinal(parser, "a,b,c\nd"); + AssertColumnEq(parser, 0, {"a", "d"}); + std::vector values; + std::vector missing; + ASSERT_OK(parser.VisitColumn( + 1, [&](const uint8_t* data, uint32_t size, bool, bool is_missing) -> Status { + values.emplace_back(reinterpret_cast(data), size); + missing.push_back(is_missing); + return Status::OK(); + })); + ASSERT_EQ(values, std::vector({"b", ""})); + ASSERT_EQ(missing, std::vector({false, true})); +} + TEST(BlockParser, EmptyHeader) { // Cannot infer number of columns uint32_t out_size;