Reject VCL OBUs with mlayer_id/tlayer_id > max

av2/decoder/decodeframe.c

@@ -7626,8 +7626,9 @@ static void handle_sequence_header(AV2Decoder *pbi, OBU_TYPE obu_type,
   }
 
   // SH activation
-  if (obu_type == OBU_CLOSED_LOOP_KEY ||
-      (obu_type == OBU_OPEN_LOOP_KEY && pbi->random_accessed)) {
+  if (keyframe_unit_in_tu &&
+      (obu_type == OBU_CLOSED_LOOP_KEY ||
+       (obu_type == OBU_OPEN_LOOP_KEY && pbi->random_accessed))) {
     pbi->active_seq[xlayer_id] = *seq_from_uch;
     // Reset malyer_id_map
     for (int i = 0; i < MAX_NUM_MLAYERS; i++)
@@ -7646,15 +7647,7 @@ static void handle_sequence_header(AV2Decoder *pbi, OBU_TYPE obu_type,
                          "Sequence Header changed at %s",
                          avm_obu_type_to_string(obu_type));
     }
-    return;
-  }
-
-  // NOTE: at this point, the current obu is first CLK/OLK in the temporal unit
-  // cm->seq_params is the currently active sequence header
-  assert(obu_type == OBU_CLOSED_LOOP_KEY || obu_type == OBU_OPEN_LOOP_KEY ||
-         obu_type == OBU_RAS_FRAME);
-
-  if (obu_type == OBU_OPEN_LOOP_KEY && !pbi->random_accessed) {
+  } else if (obu_type == OBU_OPEN_LOOP_KEY && !pbi->random_accessed) {
     if (!are_seq_headers_consistent(&cm->seq_params, seq_from_uch)) {
       avm_internal_error(&cm->error, AVM_CODEC_CORRUPT_FRAME,
                          "Sequence Header changed at OBU_OPEN_LOOP_KEY when "
@@ -7664,18 +7657,6 @@ static void handle_sequence_header(AV2Decoder *pbi, OBU_TYPE obu_type,
     }
   }
 
-  // Empty referece list
-  // NOTE: Should olk + random access reset reference list? It will be
-  // redundant but will it harm?
-  if (obu_type == OBU_CLOSED_LOOP_KEY) {
-    reset_ref_frame_map(cm);
-    for (int layer = 0; layer < MAX_NUM_MLAYERS; layer++) {
-      cm->olk_refresh_frame_flags[layer] = -1;
-      cm->olk_co_vcl_refresh_frame_flags[layer] = -1;
-      cm->prev_olk_co_vcl_refresh_frame_flags[layer] = -1;
-    }
-  }
-
   // check bitstream conformance if sequence header is parsed
   // bitstream constraint for tlayer_id
   if (cm->tlayer_id > cm->seq_params.max_tlayer_id) {
@@ -7696,6 +7677,27 @@ static void handle_sequence_header(AV2Decoder *pbi, OBU_TYPE obu_type,
         cm->mlayer_id, cm->seq_params.max_mlayer_id);
   }
 
+  if (!keyframe_unit_in_tu) {
+    return;
+  }
+
+  // NOTE: at this point, the current obu is first CLK/OLK in the temporal unit
+  // cm->seq_params is the currently active sequence header
+  assert(obu_type == OBU_CLOSED_LOOP_KEY || obu_type == OBU_OPEN_LOOP_KEY ||
+         obu_type == OBU_RAS_FRAME);
+
+  // Empty reference list
+  // NOTE: Should olk + random access reset reference list? It will be
+  // redundant but will it harm?
+  if (obu_type == OBU_CLOSED_LOOP_KEY) {
+    reset_ref_frame_map(cm);
+    for (int layer = 0; layer < MAX_NUM_MLAYERS; layer++) {
+      cm->olk_refresh_frame_flags[layer] = -1;
+      cm->olk_co_vcl_refresh_frame_flags[layer] = -1;
+      cm->prev_olk_co_vcl_refresh_frame_flags[layer] = -1;
+    }
+  }
+
   // When OBU_CONTENT_INTERPRETATION is not accompanied with the current obu
   // at a RAP boundary, cm->ci_params_per_layer[cm->mlayer_id] is reset to
   // default values and then inherited from a dependent layer.

test/assertion_test.cc

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved
+ *
+ * This source code is subject to the terms of the BSD 3-Clause Clear License
+ * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
+ * License was not distributed with this source code in the LICENSE file, you
+ * can obtain it at aomedia.org/license/software-license/bsd-3-c-c/.  If the
+ * Alliance for Open Media Patent License 1.0 was not distributed with this
+ * source code in the PATENTS file, you can obtain it at
+ * aomedia.org/license/patent-license/.
+ */
+
+#include "third_party/googletest/src/googletest/include/gtest/gtest.h"
+
+#include "config/avm_config.h"
+
+#include "avm/avmdx.h"
+#include "avm/avm_decoder.h"
+
+namespace {
+
+#if CONFIG_AV2_DECODER
+// Regression test for bug found with libavif's avif_fuzztest_dec_experimental.
+TEST(NoAssertionFailure, InvalidMlayerIdInNonKeyframeObu) {
+  // Bitstream containing:
+  // 1. OBU_SEQUENCE_HEADER (66 bytes) with max_mlayer_id = 0
+  // 2. OBU_CLOSED_LOOP_KEY (57 bytes) with mlayer_id = 0
+  // 3. OBU_SWITCH (8 bytes) with mlayer_id = 5 (> max_mlayer_id)
+  static const uint8_t kBitstream[] = {
+    0x41, 0x04, 0x85, 0xf6, 0x1c, 0x1c, 0x94, 0x98, 0x93, 0x08, 0xa8, 0xb1,
+    0xc0, 0xb5, 0x7e, 0x01, 0xec, 0xa1, 0x40, 0x77, 0x28, 0x55, 0x37, 0x45,
+    0x00, 0x03, 0xec, 0x1a, 0xeb, 0x2d, 0x2c, 0x5f, 0xcd, 0x00, 0x1f, 0x3a,
+    0xd0, 0x3f, 0xe8, 0xf6, 0x7d, 0x2a, 0x0d, 0xd1, 0x51, 0x40, 0x12, 0x00,
+    0x32, 0x23, 0xc5, 0x28, 0x04, 0xe0, 0x40, 0x00, 0x00, 0x23, 0x43, 0x30,
+    0x00, 0x42, 0xc1, 0x77, 0x00, 0x80, 0x38, 0x10, 0xf5, 0x91, 0xe5, 0x20,
+    0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x9d, 0x5c, 0x09, 0x01,
+    0x8c, 0x1a, 0x4b, 0x50, 0x44, 0x00, 0x37, 0x0c, 0x8d, 0xd6, 0x84, 0x05,
+    0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x05, 0x04, 0x04, 0x04, 0x04,
+    0x71, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04,
+    0x04, 0x05, 0x04, 0x07, 0xa8, 0xa0, 0xfc, 0x4e, 0x08, 0x04, 0x04,
+  };
+
+  avm_codec_ctx_t dec;
+  ASSERT_EQ(AVM_CODEC_OK,
+            avm_codec_dec_init(&dec, avm_codec_av2_dx(), NULL, 0));
+  EXPECT_EQ(AVM_CODEC_UNSUP_BITSTREAM,
+            avm_codec_decode(&dec, kBitstream, sizeof(kBitstream), NULL));
+  EXPECT_EQ(AVM_CODEC_OK, avm_codec_destroy(&dec));
+}
+#endif  // CONFIG_AV2_DECODER
+
+}  // namespace

test/test.cmake

@@ -50,6 +50,7 @@ list(
 list(
   APPEND
   AVM_UNIT_TEST_DECODER_SOURCES
+  "${AVM_ROOT}/test/assertion_test.cc"
   "${AVM_ROOT}/test/decode_api_test.cc"
   "${AVM_ROOT}/test/external_frame_buffer_test.cc"
   "${AVM_ROOT}/test/invalid_file_test.cc"