diff --git a/src/layers/flash_attention.cc b/src/layers/flash_attention.cc
index fb13b90fc..05812456e 100644
--- a/src/layers/flash_attention.cc
+++ b/src/layers/flash_attention.cc
@@ -110,7 +110,9 @@ namespace ctranslate2 {
 
       // init output
       StorageView context(dtype, device);
-      ops::FlashAttention fl_attn_ops(_queries_scale, _sliding_window);
+      // Causal masking only applies to decoder self-attention; encoder
+      // self-attention must attend bidirectionally.
+      ops::FlashAttention fl_attn_ops(_queries_scale, _sliding_window, /*is_causal=*/_is_decoder);
       fl_attn_ops(queries_proj, keys_proj, values_proj, context, cached_keys, cached_values, attention,
                   return_normalized_attention, rotary_cos, rotary_sin, rotary_interleaved, nullptr/*alibli*/, offset);
 
diff --git a/src/layers/whisper.cc b/src/layers/whisper.cc
index 401c14677..763b47b28 100644
--- a/src/layers/whisper.cc
+++ b/src/layers/whisper.cc
@@ -17,7 +17,8 @@ namespace ctranslate2 {
                                                                  scope + "/layer",
                                                                  _num_heads,
                                                                  /*pre_norm=*/true,
-                                                                 ops::ActivationType::GELU))
+                                                                 ops::ActivationType::GELU,
+                                                                 model.use_flash_attention()))
       , _output_norm(model, scope + "/layer_norm")
     {
     }