Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions tests/test-hrx-ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6448,6 +6448,16 @@ static void add_cases(test_runner::Suite & suite) {
// "it is normalised per block and bounded by max|V|" for good.
add(suite, "flash_attention.accumulator_range.128.1024.v60000",
[=] { run_flash_attention_accumulator_range_case(128, 1024, 60000.0f); });

// engine#2374 item 2: the same case at a smaller magnitude, because the defect has a regime the committed
// case cannot see. Measured on gfx1151 at a5cf1f2c4: V=100 passes, V=500 passes, V=60000 passes (the merged
// online_softmax floor), and **V=1000 fails with a FINITE output** -- `|v - value_magnitude| <= 5.0e-2f`
// misses while nothing is non-finite. That band is the dangerous one: a NaN aborts a run and is reported,
// a plausible wrong number is not. Pinned as known-fail so the suite stays green, and so the fix turns it
// into an UNEXPECTED-PASS instead of nobody noticing.
suite.known_fail_device_case("flash_attention.accumulator_range.128.1024.v1000",
"engine#2374 item 2: the f16 PV accumulator is finite and wrong at V=1000",
[=] { run_flash_attention_accumulator_range_case(128, 1024, 1000.0f); });
add(suite, "flash_attention.64.2.65.1/sqrt(static_cast<float>(64)).flash_attention_decode_split_f32_f16_wmma",
[=] { run_common_flash_attention_cpu_reference_case(64,
2,
Expand Down
55 changes: 45 additions & 10 deletions tests/testing_suite.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ struct Case {
std::string name;
std::function<void()> run;
bool requires_device = true;
// A case that is known to fail carries the issue it is waiting on. It does not fail the run, and a case
// whose label is set but which passes is an UNEXPECTED-PASS that does -- the same semantics the sibling
// suite has had since engine#449 (tests/hrx-backend-test.cpp), moved here so every suite that uses this
// runner can pin a known defect instead of leaving it out of the suite entirely.
const char * known_fail = nullptr;
};

enum class OutputMode {
Expand Down Expand Up @@ -80,6 +85,7 @@ struct Options {

struct Result {
std::string name;
const char * known_fail = nullptr;
std::string output;
int exit_code = -1;
int signal_code = 0;
Expand Down Expand Up @@ -255,7 +261,8 @@ inline int run_single_case_in_process(const Config & config,

inline Result run_single_case_child(const Config & config, const Options & options, const Case & test_case) {
Result result;
result.name = test_case.name;
result.name = test_case.name;
result.known_fail = test_case.known_fail;

const auto start = std::chrono::steady_clock::now();

Expand Down Expand Up @@ -388,8 +395,13 @@ inline int run_cases_parallel(const Config & config,
const size_t done = finished.fetch_add(1) + 1;
{
std::lock_guard<std::mutex> lock(print_mutex);
std::printf("%s %4zu/%zu %-86s %8lld ms\n", result.passed ? "[PASS]" : "[FAIL]", done,
const char * verdict = result.passed ? (result.known_fail != nullptr ? "[UNEXPECTED-PASS]" : "[PASS]")
: (result.known_fail != nullptr ? "[KNOWN-FAIL]" : "[FAIL]");
std::printf("%s %4zu/%zu %-86s %8lld ms\n", verdict, done,
selected_cases.size(), result.name.c_str(), static_cast<long long>(result.elapsed_ms));
if (result.known_fail != nullptr) {
std::printf(" known to fail: %s\n", result.known_fail);
}
if (!result.passed) {
if (result.signal_code != 0) {
std::printf(" terminated by signal %d\n", result.signal_code);
Expand Down Expand Up @@ -419,27 +431,43 @@ inline int run_cases_parallel(const Config & config,
thread.join();
}

size_t passed = 0;
size_t passed = 0, failed = 0, known = 0, unexpected = 0;
for (const Result & result : results) {
if (result.passed) {
++passed;
if (result.known_fail == nullptr) {
if (result.passed) {
++passed;
} else {
++failed;
}
} else if (result.passed) {
++unexpected;
} else {
++known;
}
}
const size_t failed = results.size() - passed;
const int64_t elapsed_ms =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - suite_start).count();

std::printf("\n%s summary: %zu passed, %zu failed, %zu total, %lld ms\n", config.suite_name.c_str(), passed, failed,
std::printf("\n%s summary: %zu passed, %zu failed, %zu known-fail, %zu unexpected-pass, %zu total, %lld ms\n",
config.suite_name.c_str(), passed, failed, known, unexpected,
results.size(), static_cast<long long>(elapsed_ms));
if (failed != 0) {
if (failed != 0 || unexpected != 0) {
std::printf("Failed cases:\n");
for (const Result & result : results) {
if (!result.passed) {
if (!result.passed && result.known_fail == nullptr) {
std::printf(" %s\n", result.name.c_str());
}
}
}
return failed == 0 ? 0 : 1;
if (unexpected != 0) {
std::printf("Unexpected passes - the defect these cases name is fixed, remove their known_fail label:\n");
for (const Result & result : results) {
if (result.passed && result.known_fail != nullptr) {
std::printf(" %s (%s)\n", result.name.c_str(), result.known_fail);
}
}
}
return (failed + unexpected) == 0 ? 0 : 1;
}

inline bool select_cases(const std::vector<Case> & cases,
Expand Down Expand Up @@ -501,6 +529,13 @@ class Suite {
add_case(cases_, std::move(name), std::function<void()>(std::forward<F>(run)), false);
}

// A case that fails today and names the issue it is waiting on: it is reported, counted as known-fail,
// and the run stays green. When it starts passing it is an UNEXPECTED-PASS and does fail the run.
template <typename F> void known_fail_device_case(std::string name, const char * reason, F && run) {
add_case(cases_, std::move(name), std::function<void()>(std::forward<F>(run)), true);
cases_.back().known_fail = reason;
}

bool parse_options(int argc, char ** argv, Options & options) const {
return test_runner::parse_options(argc, argv, config_, options);
}
Expand Down