From 6153bf0a3f913a7d6c196e4c431a81cc31b0859a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 30 Apr 2025 10:49:09 -0500 Subject: [PATCH 01/12] Use peak memory usage as a better proxy for ctest parallelism --- cpp/tests/CMakeLists.txt | 40 ++++++---------------------------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 06c0462ebed9..f1d90e70ca36 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -163,8 +163,6 @@ ConfigureTest( groupby/tdigest_tests.cpp groupby/topk_tests.cpp groupby/var_tests.cpp - GPUS 1 - PERCENT 100 ) # ################################################################################################## @@ -213,17 +211,13 @@ ConfigureTest( ConfigureTest( PARTITIONING_TEST partitioning/hash_partition_test.cpp partitioning/round_robin_test.cpp partitioning/partition_test.cpp - GPUS 1 - PERCENT 70 ) # ################################################################################################## # * quantiles tests ------------------------------------------------------------------------------- ConfigureTest( QUANTILES_TEST quantiles/percentile_approx_test.cpp quantiles/quantile_test.cpp - quantiles/quantiles_test.cpp - GPUS 1 - PERCENT 70 EXTRA_LIBS ${ARROW_LIBRARIES} + quantiles/quantiles_test.cpp EXTRA_LIBS ${ARROW_LIBRARIES} ) # ################################################################################################## @@ -243,8 +237,6 @@ ConfigureTest( reductions/segmented_reduction_tests.cpp reductions/tdigest_tests.cpp reductions/unique_count_tests.cpp - GPUS 1 - PERCENT 70 ) # ################################################################################################## @@ -280,6 +272,8 @@ ConfigureTest( binaryop/binop-compiled-test.cpp binaryop/binop-compiled-fixed_point-test.cpp binaryop/binop-generic-ptx-test.cpp + GPUS 1 + PERCENT 100 ) # ################################################################################################## @@ -323,11 +317,7 @@ ConfigureTest(COMPRESSION_TEST io/comp/comp_test.cpp) ConfigureTest(ROW_SELECTION_TEST io/row_selection_test.cpp) ConfigureTest(FILEPATH_SOURCE_TEST io/filepath_source_test.cpp) -ConfigureTest( - CSV_TEST io/csv_test.cpp - GPUS 1 - PERCENT 30 -) +ConfigureTest(CSV_TEST io/csv_test.cpp) ConfigureTest( ORC_TEST io/orc_chunked_reader_test.cu io/orc_test.cpp GPUS 1 @@ -377,8 +367,6 @@ ConfigureTest( ConfigureTest( JSON_TEST io/json/json_chunked_reader.cpp io/json/json_test.cpp io/json/json_type_cast_test.cpp io/json/json_utils.cu - GPUS 1 - PERCENT 30 ) ConfigureTest(JSON_WRITER_TEST io/json/json_writer.cpp) ConfigureTest(NESTED_JSON_TEST io/json/nested_json_test.cpp io/json/json_tree.cpp) @@ -386,11 +374,7 @@ ConfigureTest(MULTIBYTE_SPLIT_TEST io/text/multibyte_split_test.cpp) ConfigureTest(JSON_QUOTE_NORMALIZATION_TEST io/json/json_quote_normalization_test.cpp) ConfigureTest(JSON_WHITESPACE_NORMALIZATION_TEST io/json/json_whitespace_normalization_test.cpp) ConfigureTest(JSON_TREE_CSR_TEST io/json/json_tree_csr.cu) -ConfigureTest( - DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp - GPUS 1 - PERCENT 100 -) +ConfigureTest(DATA_CHUNK_SOURCE_TEST io/text/data_chunk_source_test.cpp) target_link_libraries(DATA_CHUNK_SOURCE_TEST PRIVATE ZLIB::ZLIB) ConfigureTest(LOGICAL_STACK_TEST io/fst/logical_stack_test.cu) ConfigureTest(FST_TEST io/fst/fst_test.cu) @@ -402,8 +386,6 @@ ConfigureTest(TYPE_INFERENCE_TEST io/type_inference_test.cpp) ConfigureTest( SORT_TEST sort/rank_test.cpp sort/segmented_sort_tests.cpp sort/sort_nested_types_tests.cpp sort/sort_test.cpp sort/stable_sort_tests.cpp sort/top_k_tests.cpp - GPUS 1 - PERCENT 70 ) # ################################################################################################## @@ -486,11 +468,7 @@ ConfigureTest(DEVICE_ATOMICS_TEST device_atomics/device_atomics_test.cu) # ################################################################################################## # * transpose tests ------------------------------------------------------------------------------- -ConfigureTest( - TRANSPOSE_TEST transpose/transpose_test.cpp - GPUS 1 - PERCENT 70 -) +ConfigureTest(TRANSPOSE_TEST transpose/transpose_test.cpp) # ################################################################################################## # * table tests ----------------------------------------------------------------------------------- @@ -541,8 +519,6 @@ ConfigureTest( rolling/range_window_bounds_test.cpp rolling/range_window_type_test.cpp rolling/rolling_test.cpp - GPUS 1 - PERCENT 70 ) # ################################################################################################## @@ -631,8 +607,6 @@ ConfigureTest( large_strings/reshape_tests.cpp large_strings/split_strings_tests.cpp io/json/json_utils.cu - GPUS 1 - PERCENT 100 ) # ################################################################################################## @@ -744,8 +718,6 @@ ConfigureTest( lists/sort_lists_tests.cpp lists/stream_compaction/apply_boolean_mask_tests.cpp lists/stream_compaction/distinct_tests.cpp - GPUS 1 - PERCENT 70 ) # ################################################################################################## From 3294b4455503fb7809e8bd6b68fbc579f4cbca32 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 30 Apr 2025 10:58:09 -0500 Subject: [PATCH 02/12] Use 100% for LARGE_STRINGS_TEST --- cpp/tests/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index f1d90e70ca36..053ab7f44f28 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -607,6 +607,8 @@ ConfigureTest( large_strings/reshape_tests.cpp large_strings/split_strings_tests.cpp io/json/json_utils.cu + GPUS 1 + PERCENT 100 ) # ################################################################################################## From 0a9147122ef2de2b45390672a9b33f2b352bb6ca Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 30 Apr 2025 15:24:33 -0500 Subject: [PATCH 03/12] Try allowing 14 tests to run simultaneously. --- cpp/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 053ab7f44f28..1f48f8e521c5 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -24,7 +24,7 @@ function(ConfigureTest CMAKE_TEST_NAME) if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT) set(_CUDF_TEST_GPUS 1) - set(_CUDF_TEST_PERCENT 15) + set(_CUDF_TEST_PERCENT 7) endif() if(NOT DEFINED _CUDF_TEST_GPUS) From 92e5eda6a8df5dc27ce7b4707d4598d6ad0da901 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 30 Apr 2025 16:13:04 -0500 Subject: [PATCH 04/12] Try 10% --- cpp/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 1f48f8e521c5..e4e0b625d397 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -24,7 +24,7 @@ function(ConfigureTest CMAKE_TEST_NAME) if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT) set(_CUDF_TEST_GPUS 1) - set(_CUDF_TEST_PERCENT 7) + set(_CUDF_TEST_PERCENT 10) endif() if(NOT DEFINED _CUDF_TEST_GPUS) From 32f51b6e2a5925935317d9f54d0feedffa40d2dc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 30 Apr 2025 22:08:50 -0500 Subject: [PATCH 05/12] Try 15% default, 70% for most of the large tests. --- cpp/tests/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index e4e0b625d397..ff2d62e0ce2f 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -24,7 +24,7 @@ function(ConfigureTest CMAKE_TEST_NAME) if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT) set(_CUDF_TEST_GPUS 1) - set(_CUDF_TEST_PERCENT 10) + set(_CUDF_TEST_PERCENT 15) endif() if(NOT DEFINED _CUDF_TEST_GPUS) @@ -273,7 +273,7 @@ ConfigureTest( binaryop/binop-compiled-fixed_point-test.cpp binaryop/binop-generic-ptx-test.cpp GPUS 1 - PERCENT 100 + PERCENT 70 ) # ################################################################################################## @@ -321,7 +321,7 @@ ConfigureTest(CSV_TEST io/csv_test.cpp) ConfigureTest( ORC_TEST io/orc_chunked_reader_test.cu io/orc_test.cpp GPUS 1 - PERCENT 100 + PERCENT 70 ) ConfigureTest( PARQUET_TEST @@ -336,7 +336,7 @@ ConfigureTest( io/parquet_v2_test.cpp io/parquet_writer_test.cpp GPUS 1 - PERCENT 100 + PERCENT 70 ) ConfigureTest( HYBRID_SCAN_TEST @@ -416,7 +416,7 @@ ConfigureTest( copying/utility_tests.cpp copying/reverse_tests.cpp GPUS 1 - PERCENT 100 + PERCENT 70 ) # ################################################################################################## From deba39730c41986584408187aef2e14ca0524fca Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 30 Apr 2025 22:17:21 -0500 Subject: [PATCH 06/12] Use async and 14-way parallelism for small tests. --- cpp/include/cudf_test/testing_main.hpp | 2 +- cpp/tests/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index 69a66ca568db..4e25487db84f 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -156,7 +156,7 @@ inline auto parse_cudf_test_opts(int argc, char** argv) std::getenv("GTEST_CUDF_STREAM_MODE"); // Overridden by CLI options char const* env_stream_error_mode = std::getenv("GTEST_CUDF_STREAM_ERROR_MODE"); // Overridden by CLI options - auto default_rmm_mode = env_rmm_mode ? env_rmm_mode : "pool"; + auto default_rmm_mode = env_rmm_mode ? env_rmm_mode : "async"; auto default_stream_mode = env_stream_mode ? env_stream_mode : "default"; auto default_stream_error_mode = env_stream_error_mode ? env_stream_error_mode : "error"; options.allow_unrecognised_options().add_options()( diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index ff2d62e0ce2f..c58bffe2a0fe 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -24,7 +24,7 @@ function(ConfigureTest CMAKE_TEST_NAME) if(NOT DEFINED _CUDF_TEST_GPUS AND NOT DEFINED _CUDF_TEST_PERCENT) set(_CUDF_TEST_GPUS 1) - set(_CUDF_TEST_PERCENT 15) + set(_CUDF_TEST_PERCENT 7) endif() if(NOT DEFINED _CUDF_TEST_GPUS) From df522ef631325f25516e132599294efc992444ee Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 9 Oct 2025 09:24:59 -0500 Subject: [PATCH 07/12] Output tests using more than 1 GB of memory. --- cpp/scripts/gtest_memory_usage.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cpp/scripts/gtest_memory_usage.sh b/cpp/scripts/gtest_memory_usage.sh index bc85f3186f60..4f3399cfbee3 100755 --- a/cpp/scripts/gtest_memory_usage.sh +++ b/cpp/scripts/gtest_memory_usage.sh @@ -5,13 +5,32 @@ export GTEST_CUDF_RMM_MODE=cuda export GTEST_CUDF_MEMORY_PEAK=1 export GTEST_BRIEF=1 + +# Collect all test results +results=() for gt in gtests/*_TEST ; do test_name=$(basename "${gt}") echo -n "$test_name" # dependent on the string output from testing_main.hpp bytes=$(${gt} 2>/dev/null | grep Peak | cut -d' ' -f4) echo ",${bytes}" + results+=("$test_name,$bytes") done + unset GTEST_BRIEF unset GTEST_CUDF_RMM_MODE unset GTEST_CUDF_MEMORY_PEAK + +# Output tests using more than 1GB +echo "" +echo "Tests using more than 1GB of memory:" +threshold=1073741824 # 1GB in bytes +for result in "${results[@]}" ; do + test_name=$(echo "$result" | cut -d',' -f1) + bytes=$(echo "$result" | cut -d',' -f2) + if [[ -n "$bytes" && "$bytes" -gt "$threshold" ]] ; then + # Convert bytes to GB with 2 decimal places + gb=$(awk "BEGIN {printf \"%.2f\", $bytes / 1073741824}") + echo "$test_name: ${gb} GB" + fi +done From 1e35992270c4c97ae28eab3f63f4abb2b0f03e44 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 9 Oct 2025 09:28:51 -0500 Subject: [PATCH 08/12] Add note about memory usage --- cpp/tests/CMakeLists.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index c58bffe2a0fe..d400ef16ca10 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -14,6 +14,29 @@ include(rapids-test) rapids_test_init() rapids_cmake_install_lib_dir(lib_dir) +# ################################################################################################## +# **Note about test memory usage:** +# +# We design the test suite to run in CI with max parallelism on GPUs with 16 GB of memory or more. +# Tests are assigned GPU memory percentages to enable safe parallel execution. We draw an arbitrary +# line that tests are allowed to run in parallel if they consume less than 1 GB of peak GPU memory, +# and are run serially otherwise. This allows up to 14 tests in parallel on a 16 GB GPU with a bit +# of room for overhead. (14 * 7% is 98%, and we are required to use integer percentages.) Therefore, +# tests are assigned 7% of the GPU by default, and 100% if they use >=1 GB of memory. ctest won't +# schedule more than 100% of the GPU, so at most 14 tests in parallel. This means on a 16 GB GPU, we +# can safely run `ctest -jN` with N>=14. +# +# **Guidelines:** +# +# * Tests using <=1 GB: Use default (no GPUS/PERCENT parameters, defaults to 7%) +# * Tests using >1 GB: Mark as "GPUS 1 PERCENT 100" to reserve the entire GPU +# +# Marking large tests with PERCENT 100 prevents memory oversubscription and ensures only one large +# test runs at a time, while smaller tests can still run in parallel. +# +# The peak memory usage of all tests can be determined with cpp/scripts/gtest_memory_usage.sh. +# ################################################################################################## + # This function takes in a test name and test source and handles setting all of the associated # properties and linking to build the test function(ConfigureTest CMAKE_TEST_NAME) From d3d7e3fe4b22588000f33dab8343e9bdeb0e7e46 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 9 Oct 2025 09:29:09 -0500 Subject: [PATCH 09/12] Update PERCENT 100 based on memory usage --- cpp/tests/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index d400ef16ca10..0c9cc52ba326 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -296,7 +296,7 @@ ConfigureTest( binaryop/binop-compiled-fixed_point-test.cpp binaryop/binop-generic-ptx-test.cpp GPUS 1 - PERCENT 70 + PERCENT 100 ) # ################################################################################################## @@ -344,7 +344,7 @@ ConfigureTest(CSV_TEST io/csv_test.cpp) ConfigureTest( ORC_TEST io/orc_chunked_reader_test.cu io/orc_test.cpp GPUS 1 - PERCENT 70 + PERCENT 100 ) ConfigureTest( PARQUET_TEST @@ -359,7 +359,7 @@ ConfigureTest( io/parquet_v2_test.cpp io/parquet_writer_test.cpp GPUS 1 - PERCENT 70 + PERCENT 100 ) ConfigureTest( HYBRID_SCAN_TEST @@ -439,7 +439,7 @@ ConfigureTest( copying/utility_tests.cpp copying/reverse_tests.cpp GPUS 1 - PERCENT 70 + PERCENT 100 ) # ################################################################################################## @@ -562,6 +562,8 @@ ConfigureTest( ConfigureTest( RESHAPE_TEST reshape/byte_cast_tests.cpp reshape/interleave_columns_tests.cpp reshape/table_to_array_tests.cpp reshape/tile_tests.cpp + GPUS 1 + PERCENT 100 ) # ################################################################################################## From de1201c0b4facb90be436a57951dc1b360e49a55 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 9 Oct 2025 10:37:19 -0500 Subject: [PATCH 10/12] Rerun CI From 6edeb0152a67812d529a88bf5defde62ec627a97 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 18 Aug 2026 14:43:42 -0500 Subject: [PATCH 11/12] Apply suggestions from code review Co-authored-by: Lawrence Mitchell --- cpp/scripts/gtest_memory_usage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/scripts/gtest_memory_usage.sh b/cpp/scripts/gtest_memory_usage.sh index 4f3399cfbee3..5395d5af7c4e 100755 --- a/cpp/scripts/gtest_memory_usage.sh +++ b/cpp/scripts/gtest_memory_usage.sh @@ -24,13 +24,13 @@ unset GTEST_CUDF_MEMORY_PEAK # Output tests using more than 1GB echo "" echo "Tests using more than 1GB of memory:" -threshold=1073741824 # 1GB in bytes +threshold=$((1024 * 1024 * 1024)) for result in "${results[@]}" ; do test_name=$(echo "$result" | cut -d',' -f1) bytes=$(echo "$result" | cut -d',' -f2) if [[ -n "$bytes" && "$bytes" -gt "$threshold" ]] ; then # Convert bytes to GB with 2 decimal places - gb=$(awk "BEGIN {printf \"%.2f\", $bytes / 1073741824}") + gb=$(awk "BEGIN {printf \"%.2f\", $bytes / (1024 * 1024 * 1024)}") echo "$test_name: ${gb} GB" fi done From a5a9e7741865367164b7d58f81bf97515afa7e59 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 18 Aug 2026 22:42:32 -0500 Subject: [PATCH 12/12] Update copyright notices --- cpp/include/cudf_test/testing_main.hpp | 2 +- cpp/scripts/gtest_memory_usage.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/cudf_test/testing_main.hpp b/cpp/include/cudf_test/testing_main.hpp index 4e25487db84f..fca30024a1da 100644 --- a/cpp/include/cudf_test/testing_main.hpp +++ b/cpp/include/cudf_test/testing_main.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/scripts/gtest_memory_usage.sh b/cpp/scripts/gtest_memory_usage.sh index 5395d5af7c4e..1c48cabcdc96 100755 --- a/cpp/scripts/gtest_memory_usage.sh +++ b/cpp/scripts/gtest_memory_usage.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 export GTEST_CUDF_RMM_MODE=cuda