diff --git a/av2/encoder/speed_features.c b/av2/encoder/speed_features.c
index b4e9d66d9e..bb8a97e475 100644
--- a/av2/encoder/speed_features.c
+++ b/av2/encoder/speed_features.c
@@ -461,11 +461,15 @@ static void set_good_speed_features_framesize_independent(
     sf->mv_sf.reduce_search_range = 1;
     sf->mv_sf.subpel_search_type = boosted ? USE_8_TAPS : USE_4_TAPS;
     sf->mv_sf.subpel_iters_per_step = boosted ? 2 : 1;
+    sf->mv_sf.warp_mv_refine_early_term =
+        (cm->current_frame.pyramid_level >= 3);
+
     sf->lpf_sf.ccso_chroma_dep = 1;
   }
 
   if (speed >= 2) {
     sf->mv_sf.warp_search_method_sec_ref = WARP_SEARCH_DIAMOND;
+    sf->mv_sf.warp_mv_refine_early_term = boosted ? false : true;
 
     sf->lpf_sf.early_terminate_ccso_search_by_cost = 1;
     sf->part_sf.partition_pruning_with_mlp_none_thresh = 2.5f;
diff --git a/av2/encoder/speed_features.h b/av2/encoder/speed_features.h
index 66059ae249..5694877fcb 100644
--- a/av2/encoder/speed_features.h
+++ b/av2/encoder/speed_features.h
@@ -566,7 +566,10 @@ typedef struct MV_SPEED_FEATURES {
   int warp_search_iters;
 
   // Early-terminate warp MV refinement when the RD improvement over the
-  // previous iteration is less than ~5%. Enabled for speed >= 3.
+  // previous iteration is less than ~5%.
+  // For speed = 1, enabled for frames with pyramid_level >= 3.
+  // For speed = 2, enabled for frames with pyramid_level >= 2.
+  // For speed >= 3, enabled for all frames.
   bool warp_mv_refine_early_term;
 
   // Use faster motion search settings for partition blocks with at least one