diff --git a/av2/encoder/encode_strategy.c b/av2/encoder/encode_strategy.c
index 6e6eb63d39..73510db30d 100644
--- a/av2/encoder/encode_strategy.c
+++ b/av2/encoder/encode_strategy.c
@@ -794,6 +794,7 @@ int av2_get_refresh_frame_flags(
           // been output yet and whose DOH is at least the current
           // frame's DOH. (DOH requirement)
           if (cm->ref_frame_map[i] != NULL &&
+              !cm->ref_frame_map[i]->is_restricted &&
               cm->ref_frame_map[i]->implicit_output_picture &&
               !cm->ref_frame_map[i]->frame_output_done &&
               (int)cm->ref_frame_map[i]->display_order_hint >= cur_disp_order) {
@@ -847,7 +848,7 @@ int av2_get_refresh_frame_flags(
   // at least the current frame's DOH. (DOH requirement)
   for (int i = 0; i < cpi->common.seq_params.ref_frames; i++) {
     const RefCntBuffer *const buf = cpi->common.ref_frame_map[i];
-    if (buf != NULL && buf->implicit_output_picture &&
+    if (buf != NULL && !buf->is_restricted && buf->implicit_output_picture &&
         !buf->frame_output_done &&
         (int)buf->display_order_hint >= cur_disp_order) {
       olk_flags_to_keep |= (1 << i);
diff --git a/av2/encoder/encoder.c b/av2/encoder/encoder.c
index e0d0376d4d..8ea07bfeb4 100644
--- a/av2/encoder/encoder.c
+++ b/av2/encoder/encoder.c
@@ -5116,6 +5116,7 @@ int av2_encode(AV2_COMP *const cpi, uint8_t *const dest,
                                                cm->ref_frame_map[i]->mlayer_id,
                                                cm->mlayer_id)) {
             cm->ref_frame_map[i]->is_restricted = true;
+            cm->ref_frame_map[i]->display_order_hint = REF_RESTRICTED_DOH;
           }
         }
       }
diff --git a/av2/encoder/rdopt.h b/av2/encoder/rdopt.h
index 4607390495..c1d4124a77 100644
--- a/av2/encoder/rdopt.h
+++ b/av2/encoder/rdopt.h
@@ -215,13 +215,6 @@ static INLINE int prune_ref_by_selective_ref_frame(
   assert(ref_frame[0] != NONE_FRAME);
   if (ref_frame[0] == INTRA_FRAME) return 0;
 
-  RefCntBuffer *ref0_buf = get_ref_frame_buf(cm, ref_frame[0]);
-  if (ref0_buf)
-    if (ref0_buf->is_restricted) return 1;
-  RefCntBuffer *ref1_buf = get_ref_frame_buf(cm, ref_frame[1]);
-  if (ref1_buf)
-    if (ref1_buf->is_restricted) return 1;
-
   const int comp_pred = is_inter_ref_frame(ref_frame[1]);
 
   if (comp_pred && ref_frame[0] >= RANKED_REF0_TO_PRUNE) return 1;
@@ -259,6 +252,13 @@ static INLINE int prune_ref_by_selective_ref_frame(
     }
   }
 
+  RefCntBuffer *ref0_buf = get_ref_frame_buf(cm, ref_frame[0]);
+  if (ref0_buf)
+    if (ref0_buf->is_restricted) return 0;
+  RefCntBuffer *ref1_buf = get_ref_frame_buf(cm, ref_frame[1]);
+  if (ref1_buf)
+    if (ref1_buf->is_restricted) return 0;
+
   int dir_refrank0[2] = { -1, -1 };
   int dir_refrank1[2] = { -1, -1 };
   int d0 = get_dir_rank(cm, ref_frame[0], dir_refrank0);