diff --git a/tests/test-hrx-ops.cpp b/tests/test-hrx-ops.cpp index b1e6ceaef837..79c0f87e93b6 100644 --- a/tests/test-hrx-ops.cpp +++ b/tests/test-hrx-ops.cpp @@ -6448,6 +6448,16 @@ static void add_cases(test_runner::Suite & suite) { // "it is normalised per block and bounded by max|V|" for good. add(suite, "flash_attention.accumulator_range.128.1024.v60000", [=] { run_flash_attention_accumulator_range_case(128, 1024, 60000.0f); }); + + // engine#2374 item 2: the same case at a smaller magnitude, because the defect has a regime the committed + // case cannot see. Measured on gfx1151 at a5cf1f2c4: V=100 passes, V=500 passes, V=60000 passes (the merged + // online_softmax floor), and **V=1000 fails with a FINITE output** -- `|v - value_magnitude| <= 5.0e-2f` + // misses while nothing is non-finite. That band is the dangerous one: a NaN aborts a run and is reported, + // a plausible wrong number is not. Pinned as known-fail so the suite stays green, and so the fix turns it + // into an UNEXPECTED-PASS instead of nobody noticing. + suite.known_fail_device_case("flash_attention.accumulator_range.128.1024.v1000", + "engine#2374 item 2: the f16 PV accumulator is finite and wrong at V=1000", + [=] { run_flash_attention_accumulator_range_case(128, 1024, 1000.0f); }); add(suite, "flash_attention.64.2.65.1/sqrt(static_cast(64)).flash_attention_decode_split_f32_f16_wmma", [=] { run_common_flash_attention_cpu_reference_case(64, 2, diff --git a/tests/testing_suite.h b/tests/testing_suite.h index e1c9dc43d6c2..92333763231a 100644 --- a/tests/testing_suite.h +++ b/tests/testing_suite.h @@ -28,6 +28,11 @@ struct Case { std::string name; std::function run; bool requires_device = true; + // A case that is known to fail carries the issue it is waiting on. It does not fail the run, and a case + // whose label is set but which passes is an UNEXPECTED-PASS that does -- the same semantics the sibling + // suite has had since engine#449 (tests/hrx-backend-test.cpp), moved here so every suite that uses this + // runner can pin a known defect instead of leaving it out of the suite entirely. + const char * known_fail = nullptr; }; enum class OutputMode { @@ -80,6 +85,7 @@ struct Options { struct Result { std::string name; + const char * known_fail = nullptr; std::string output; int exit_code = -1; int signal_code = 0; @@ -255,7 +261,8 @@ inline int run_single_case_in_process(const Config & config, inline Result run_single_case_child(const Config & config, const Options & options, const Case & test_case) { Result result; - result.name = test_case.name; + result.name = test_case.name; + result.known_fail = test_case.known_fail; const auto start = std::chrono::steady_clock::now(); @@ -388,8 +395,13 @@ inline int run_cases_parallel(const Config & config, const size_t done = finished.fetch_add(1) + 1; { std::lock_guard lock(print_mutex); - std::printf("%s %4zu/%zu %-86s %8lld ms\n", result.passed ? "[PASS]" : "[FAIL]", done, + const char * verdict = result.passed ? (result.known_fail != nullptr ? "[UNEXPECTED-PASS]" : "[PASS]") + : (result.known_fail != nullptr ? "[KNOWN-FAIL]" : "[FAIL]"); + std::printf("%s %4zu/%zu %-86s %8lld ms\n", verdict, done, selected_cases.size(), result.name.c_str(), static_cast(result.elapsed_ms)); + if (result.known_fail != nullptr) { + std::printf(" known to fail: %s\n", result.known_fail); + } if (!result.passed) { if (result.signal_code != 0) { std::printf(" terminated by signal %d\n", result.signal_code); @@ -419,27 +431,43 @@ inline int run_cases_parallel(const Config & config, thread.join(); } - size_t passed = 0; + size_t passed = 0, failed = 0, known = 0, unexpected = 0; for (const Result & result : results) { - if (result.passed) { - ++passed; + if (result.known_fail == nullptr) { + if (result.passed) { + ++passed; + } else { + ++failed; + } + } else if (result.passed) { + ++unexpected; + } else { + ++known; } } - const size_t failed = results.size() - passed; const int64_t elapsed_ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - suite_start).count(); - std::printf("\n%s summary: %zu passed, %zu failed, %zu total, %lld ms\n", config.suite_name.c_str(), passed, failed, + std::printf("\n%s summary: %zu passed, %zu failed, %zu known-fail, %zu unexpected-pass, %zu total, %lld ms\n", + config.suite_name.c_str(), passed, failed, known, unexpected, results.size(), static_cast(elapsed_ms)); - if (failed != 0) { + if (failed != 0 || unexpected != 0) { std::printf("Failed cases:\n"); for (const Result & result : results) { - if (!result.passed) { + if (!result.passed && result.known_fail == nullptr) { std::printf(" %s\n", result.name.c_str()); } } } - return failed == 0 ? 0 : 1; + if (unexpected != 0) { + std::printf("Unexpected passes - the defect these cases name is fixed, remove their known_fail label:\n"); + for (const Result & result : results) { + if (result.passed && result.known_fail != nullptr) { + std::printf(" %s (%s)\n", result.name.c_str(), result.known_fail); + } + } + } + return (failed + unexpected) == 0 ? 0 : 1; } inline bool select_cases(const std::vector & cases, @@ -501,6 +529,13 @@ class Suite { add_case(cases_, std::move(name), std::function(std::forward(run)), false); } + // A case that fails today and names the issue it is waiting on: it is reported, counted as known-fail, + // and the run stays green. When it starts passing it is an UNEXPECTED-PASS and does fail the run. + template void known_fail_device_case(std::string name, const char * reason, F && run) { + add_case(cases_, std::move(name), std::function(std::forward(run)), true); + cases_.back().known_fail = reason; + } + bool parse_options(int argc, char ** argv, Options & options) const { return test_runner::parse_options(argc, argv, config_, options); }