ci: Try to reduce costs and timeouts for expensive tests in debug mode (#5150)

* Fix setting and response of OIIO_CI -- it was less functional before
than I had thought because we didn't actually define the symbol for
compilation.

* More aggressive cost reduction when combining CI + Debug builds,
reduce resolution of some benchmarks, etc., in addition to reducing
iterations as we had done before.

* Mark the most expensive unit tests with a cost, so CTest knows to
start them first. (more explanation below)

* Add Testing/Temporary/CTestCostData.txt to the cache-save paths --
that preserves CTest's understanding of how much each test costs so it
can schedule it more intellegently for the next run.

Signed-off-by: Larry Gritz <lg@larrygritz.com>

Author: lgritz

CMakeLists.txt

@@ -142,6 +142,12 @@ if (TEX_BATCH_SIZE)
     add_compile_definitions (OIIO_TEXTURE_SIMD_BATCH_WIDTH=${TEX_BATCH_SIZE})
 endif ()
 
+# If we're building this just for CI, define a symbol so the code can
+# tell (and reduce complexity of some tests)
+set_option (OIIO_CI "Set if this build is for our GHA CI" OFF)
+if (OIIO_CI)
+    add_compile_definitions (OIIO_CI=1)
+endif ()
 
 # Namespace settings
 #

src/build-scripts/ci-startup.bash

@@ -36,7 +36,7 @@ export PYTHONPATH=$OpenImageIO_ROOT/lib/python${PYTHON_VERSION}/site-packages:$P
 export COMPILER=${COMPILER:=gcc}
 export CC=${CC:=gcc}
 export CXX=${CXX:=g++}
-export OpenImageIO_CI=true
+export OpenImageIO_CI=1
 export USE_NINJA=${USE_NINJA:=1}
 export CMAKE_GENERATOR=${CMAKE_GENERATOR:=Ninja}
 export CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:=Release}

src/libOpenImageIO/CMakeLists.txt

@@ -263,11 +263,13 @@ if (OIIO_BUILD_TESTS AND BUILD_TESTING)
                           LINK_LIBRARIES OpenImageIO Imath::Imath
                           FOLDER "Unit Tests" NO_INSTALL)
     add_test (unit_image_span ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/image_span_test)
+    set_tests_properties (unit_image_span PROPERTIES COST 10)
 
     fancy_add_executable (NAME imagebuf_test SRC imagebuf_test.cpp
                           LINK_LIBRARIES OpenImageIO
                           FOLDER "Unit Tests" NO_INSTALL)
     add_test (unit_imagebuf ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/imagebuf_test)
+    set_tests_properties (unit_imagebuf PROPERTIES COST 30)
 
     fancy_add_executable (NAME imagecache_test SRC imagecache_test.cpp
                           LINK_LIBRARIES OpenImageIO
@@ -284,6 +286,7 @@ if (OIIO_BUILD_TESTS AND BUILD_TESTING)
                           FOLDER "Unit Tests" NO_INSTALL)
     target_compile_definitions (imagebufalgo_test PRIVATE OIIO_USE_HWY=${_oiio_use_hwy})
     add_test (unit_imagebufalgo ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/imagebufalgo_test)
+    set_tests_properties (unit_imagebufalgo PROPERTIES PROCESSORS 2 COST 60)
 
     fancy_add_executable (NAME imagespec_test SRC imagespec_test.cpp
                           LINK_LIBRARIES OpenImageIO
@@ -294,6 +297,7 @@ if (OIIO_BUILD_TESTS AND BUILD_TESTING)
                           LINK_LIBRARIES OpenImageIO
                           FOLDER "Unit Tests" NO_INSTALL)
     add_test (unit_imageinout ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/imageinout_test)
+    set_tests_properties (unit_imageinout PROPERTIES PROCESSORS 2 COST 30)
 
     if (NOT DEFINED ENV{OpenImageIO_CI})
         fancy_add_executable (NAME imagespeed_test SRC imagespeed_test.cpp

src/libOpenImageIO/image_span_test.cpp

@@ -21,6 +21,14 @@ using namespace OIIO;
 static int ntrials    = 5;
 static int iterations = 0;
 
+#if defined(NDEBUG) || !defined(OIIO_CI)
+static const int big_xres = 2048, big_yres = 1536;
+#else
+// Only for debug builds that are part of OIIO's CI - reduce resolution to
+// make it run faster
+static const int big_xres = 2048 / 2, big_yres = 1536 / 2;
+#endif
+
 
 
 template<typename T>
@@ -156,7 +164,7 @@ template<typename T = float>
 void
 test_image_span_copy_image()
 {
-    const int xres = 2048, yres = 1536, nchans = 4;
+    const int xres = big_xres, yres = big_yres, nchans = 4;
     const size_t chansize = sizeof(T);
     print("\nTesting copy_image {} (total {} MB):\n", TypeDescFromC<T>::value(),
           xres * yres * nchans * chansize * 3 / 4 / 1024 / 1024);
@@ -223,7 +231,7 @@ test_image_span_contiguize()
     // Benchmark old (ptr) versus new (span) contiguize functions
     using pvt::contiguize;
 
-    const int xres = 2048, yres = 1536, nchans = 4;
+    const int xres = big_xres, yres = big_yres, nchans = 4;
     const size_t chansize = sizeof(T);
     print("\nTesting contiguize {} (total {} MB):\n", TypeDescFromC<T>::value(),
           xres * yres * nchans * chansize * 3 / 4 / 1024 / 1024);
@@ -296,7 +304,7 @@ void
 test_image_span_convert_image()
 {
     // Benchmark old (ptr) versus new (span) convert_image functions
-    const int xres = 2048, yres = 1536, nchans = 4;
+    const int xres = big_xres, yres = big_yres, nchans = 4;
     const size_t schansize = sizeof(Stype);
     const size_t dchansize = sizeof(Dtype);
     print("\nTesting convert_image {} -> {} (total {}M values):\n",
@@ -423,7 +431,7 @@ void
 benchmark_image_span_passing()
 {
     print("\nbenchmark_image_span_passing\n");
-    const int xres = 2048, yres = 1536, nchans = 4;
+    const int xres = big_xres, yres = big_yres, nchans = 4;
     std::vector<float> sbuf(xres * yres * nchans, 1.0f);
     image_span<const float> ispan(sbuf.data(), nchans, xres, yres, 1);
 

src/libOpenImageIO/imagebuf_test.cpp

@@ -409,7 +409,13 @@ time_get_pixels()
     bench.trials(ntrials);
     bench.iterations(iterations);
     const int nchans = 4;
+#if defined(NDEBUG) || !defined(OIIO_CI)
     const int xres = 2000, yres = 1000;
+#else
+    // Only for debug builds that are part of OIIO's CI - reduce resolution to
+    // make it run faster
+    const int xres = 1000, yres = 500;
+#endif
     ImageBuf A(ImageSpec(xres, yres, nchans, TypeDesc::FLOAT));
     ImageBufAlgo::zero(A);
 
@@ -609,7 +615,14 @@ void
 time_iterators()
 {
     print("Timing iterator operations:\n");
+
+#if defined(NDEBUG) || !defined(OIIO_CI)
     const int rez = 4096, nchans = 4;
+#else
+    // Only for debug builds that are part of OIIO's CI - reduce resolution to
+    // make it run faster
+    const int rez = 1024, nchans = 4;
+#endif
     ImageSpec spec(rez, rez, nchans, TypeFloat);
     ImageBuf img(spec);
     ImageBufAlgo::fill(img, { 0.25f, 0.5f, 0.75f, 1.0f });

src/libOpenImageIO/imagebufalgo_test.cpp

@@ -205,10 +205,17 @@ test_zero_fill()
     Benchmarker bench;
     bench.trials(ntrials);
     bench.iterations(iterations);
-    ImageBuf buf_rgba_float(ImageSpec(1000, 1000, 4, TypeFloat));
-    ImageBuf buf_rgba_uint8(ImageSpec(1000, 1000, 4, TypeUInt8));
-    ImageBuf buf_rgba_half(ImageSpec(1000, 1000, 4, TypeHalf));
-    ImageBuf buf_rgba_uint16(ImageSpec(1000, 1000, 4, TypeDesc::UINT16));
+#if defined(NDEBUG) || !defined(OIIO_CI)
+    const int rez = 1000;
+#else
+    // Only for debug builds that are part of OIIO's CI - reduce resolution to
+    // make it run faster
+    const int rez = 256;
+#endif
+    ImageBuf buf_rgba_float(ImageSpec(rez, rez, 4, TypeFloat));
+    ImageBuf buf_rgba_uint8(ImageSpec(rez, rez, 4, TypeUInt8));
+    ImageBuf buf_rgba_half(ImageSpec(rez, rez, 4, TypeHalf));
+    ImageBuf buf_rgba_uint16(ImageSpec(rez, rez, 4, TypeDesc::UINT16));
     float vals[] = { 0, 0, 0, 0 };
     bench("  IBA::fill float[4] ",
           [&]() { ImageBufAlgo::fill(buf_rgba_float, cspan<float>(vals)); });
@@ -1413,10 +1420,17 @@ test_simple_perpixel()
         bench.trials(ntrials);
         bench.iterations(iterations);
         bench.units(Benchmarker::Unit::ms);
-        ImageBuf af(ImageSpec(2048, 2048, 4, TypeFloat));
-        ImageBuf bf(ImageSpec(2048, 2048, 4, TypeFloat));
-        ImageBuf au8(ImageSpec(2048, 2048, 4, TypeUInt8));
-        ImageBuf bu8(ImageSpec(2048, 2048, 4, TypeUInt8));
+#if defined(NDEBUG) || !defined(OIIO_CI)
+        const int rez = 2048;
+#else
+        // Only for debug builds that are part of OIIO's CI - reduce resolution to
+        // make it run faster
+        const int rez = 1024;
+#endif
+        ImageBuf af(ImageSpec(rez, rez, 4, TypeFloat));
+        ImageBuf bf(ImageSpec(rez, rez, 4, TypeFloat));
+        ImageBuf au8(ImageSpec(rez, rez, 4, TypeUInt8));
+        ImageBuf bu8(ImageSpec(rez, rez, 4, TypeUInt8));
         bench("  IBA::add() float",
               [&]() { ImageBuf r = ImageBufAlgo::add(af, bf); });
         bench("  IBA::add() u8",
@@ -1747,7 +1761,9 @@ main(int argc, char** argv)
     benchmark_parallel_image(64, iterations * 64);
     benchmark_parallel_image(512, iterations * 16);
     benchmark_parallel_image(1024, iterations * 4);
+#if defined(NDEBUG) || !defined(OIIO_CI)
     benchmark_parallel_image(2048, iterations);
+#endif
 
     return unit_test_failures;
 }

src/libutil/hash_test.cpp

@@ -11,6 +11,7 @@
 
 #include <OpenImageIO/argparse.h>
 #include <OpenImageIO/benchmark.h>
+#include <OpenImageIO/fmath.h>
 #include <OpenImageIO/hash.h>
 #include <OpenImageIO/strutil.h>
 #include <OpenImageIO/timer.h>
@@ -215,7 +216,7 @@ main(int argc, char* argv[])
 
     // fill data with random values so we can hash it a bunch of different ways
     std::mt19937 rng(42);
-    data.resize(iterations / sizeof(data[0]), 0);
+    data.resize(round_to_multiple(iterations, 4) / sizeof(data[0]), 0);
     for (uint32_t& d : data)
         d = rng();