diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/flash_attention/online_softmax.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/flash_attention/online_softmax.loom index a6ee1946480c..a986dd83f307 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/flash_attention/online_softmax.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/flash_attention/online_softmax.loom @@ -7,11 +7,23 @@ template.def<@ggml.flash_attention.online_softmax.prefill> device @ggml_flash_at %probability_stage_view = buffer.view %probability_stage[%c0_offset] : buffer -> view<16x64xf16, %probability_layout> %block_max = kernel.subgroup.reduce %masked_scores : vector<4xf32> %next_max = vector.maxnumf %current_max, %block_max : vector<4xf32> + // engine#2374 item 2: a fully masked block leaves both %current_max and %block_max at -inf, and + // %current_max - %next_max is then -inf - -inf = NaN, which %old_scale exponentiates into the running + // sum. Flooring the max at a large NEGATIVE finite value keeps the ordinary path identical (a finite + // next_max is unchanged by maxnumf) and makes the fully-masked case give exp(-inf) = 0: a first block + // contributes nothing to a sum that is already zero, which is the correct answer rather than NaN. + // guard: f16-narrowing -- declared for tools/ci/hrx_f16_narrowing.py, whose GUARD markers are + // ("scale_log2", "prescale") for the prefill GEMMs. This one is not a prescale: it floors the running max + // before the scale subtraction so a fully masked block gives exp(-inf)=0 instead of -inf - -inf = NaN. + // The declared marker is what lets the ratchet record this body, so a conflict resolution that drops the + // next line is named rather than counted. + %next_max_floor = vector.constant -3.400000e+38 : vector<4xf32> + %next_max_safe = vector.maxnumf %next_max, %next_max_floor : vector<4xf32> %score_delta = vector.subf %masked_scores, %next_max : vector<4xf32> %raw_probability = vector.expf %score_delta : vector<4xf32> %probability = vector.select %valid, %raw_probability, %c0_f32x4 : vector<4xf32> %block_sum = kernel.subgroup.reduce %probability : vector<4xf32> - %old_delta = vector.subf %current_max, %next_max : vector<4xf32> + %old_delta = vector.subf %current_max, %next_max_safe : vector<4xf32> %old_scale = vector.expf %old_delta : vector<4xf32> %scaled_current_sum = vector.mulf %current_sum, %old_scale : vector<4xf32> %next_sum = vector.addf %scaled_current_sum, %block_sum : vector<4xf32>