diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f07e07..66cfcc9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -180,6 +180,13 @@ if(BUILD_CONTAINER_TOOLS) GIT_SHALLOW FALSE # Disable shallow clone for branch GIT_SUBMODULES "" # Don't initialize any submodules during fetch UPDATE_DISCONNECTED TRUE # Don't update on every build + # Fix off-by-one in SampleToChunkAtom::lookupSample(): the boundary sample of a + # stsc run (e.g. the first sync sample of a new CVS) is misattributed to the + # previous run's sample description index, corrupting av2C config-OBU reinjection + # on demux for multi-CVS streams. + PATCH_COMMAND sed -i.bak + "s/if(sampleNumber <= (firstSampleThisChunkRun + samplesThisChunkRun))/if(sampleNumber < (firstSampleThisChunkRun + samplesThisChunkRun))/" + IsoLib/libisomediafile/src/SampleToChunkAtom.c ) # Configure isobmff to build library only (no submodules, no examples) diff --git a/apps/av2_demux/av2_demuxer.cpp b/apps/av2_demux/av2_demuxer.cpp index db38c1b..4deab3d 100644 --- a/apps/av2_demux/av2_demuxer.cpp +++ b/apps/av2_demux/av2_demuxer.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -75,13 +76,12 @@ uint32_t be32(const uint8_t* p) { (static_cast(p[2]) << 8) | static_cast(p[3]); } -bool read_av2c_payload(MP4Media media, std::vector& out_bytes) { +bool read_av2c_payload(MP4Media media, uint32_t desc_index, std::vector& out_bytes) { MP4Handle entry_h = nullptr; if (MP4NewHandle(0, &entry_h) != MP4NoErr || !entry_h) { spdlog::error("MP4NewHandle(sampleEntry) failed"); return false; } - uint32_t desc_index = 1; if (MP4GetMediaSampleDescription(media, desc_index, entry_h, /*outIdx=*/nullptr) != MP4NoErr) { spdlog::error("MP4GetMediaSampleDescription({}) failed", desc_index); MP4DisposeHandle(entry_h); @@ -154,21 +154,37 @@ bool Av2Demuxer::demux(const std::string& input_mp4, const std::string& output_o return false; } - std::vector av2c_bytes; - if (!read_av2c_payload(media, av2c_bytes)) { - MP4DisposeMovie(movie); - return false; + // Discover every stsd sample entry (one per CVS) by probing indices until failure; + // libisomedia has no MP4GetMediaSampleDescriptionCount API. + std::map> config_obus_map; + for (uint32_t desc_index = 1;; ++desc_index) { + MP4Handle probe_h = nullptr; + if (MP4NewHandle(0, &probe_h) != MP4NoErr || !probe_h) break; + MP4Err probe_err = MP4GetMediaSampleDescription(media, desc_index, probe_h, nullptr); + MP4DisposeHandle(probe_h); + if (probe_err != MP4NoErr) break; + + std::vector av2c_bytes; + if (!read_av2c_payload(media, desc_index, av2c_bytes)) { + MP4DisposeMovie(movie); + return false; + } + if (av2c_bytes.size() < kAv2CPrefixSize) { + spdlog::error("av2C payload too short ({} bytes, need >= {}) for desc_idx {}", + av2c_bytes.size(), kAv2CPrefixSize, desc_index); + MP4DisposeMovie(movie); + return false; + } + config_obus_map[desc_index] = + std::vector(av2c_bytes.begin() + kAv2CPrefixSize, av2c_bytes.end()); + spdlog::debug("av2C desc_idx {}: {} configOBU bytes", desc_index, + config_obus_map[desc_index].size()); } - if (av2c_bytes.size() < kAv2CPrefixSize) { - spdlog::error("av2C payload too short ({} bytes, need >= {})", av2c_bytes.size(), - kAv2CPrefixSize); + if (config_obus_map.empty()) { + spdlog::error("No av2C sample descriptions found"); MP4DisposeMovie(movie); return false; } - const uint8_t* config_obus = av2c_bytes.data() + kAv2CPrefixSize; - size_t config_obus_size = av2c_bytes.size() - kAv2CPrefixSize; - spdlog::debug("av2C: {} payload bytes ({} prefix + {} configOBUs)", av2c_bytes.size(), - kAv2CPrefixSize, config_obus_size); uint32_t sample_count = 0; if (MP4GetMediaSampleCount(media, &sample_count) != MP4NoErr) { @@ -185,7 +201,10 @@ bool Av2Demuxer::demux(const std::string& input_mp4, const std::string& output_o return false; } - // For every sample: emit a TD first, then (only for sample 1) the configOBUs, then the sample bytes. + // For every sample: emit a TD first, then (only for sync samples) the configOBUs for that + // sample's desc_idx, then the sample bytes. AV2's SH is prohibited from appearing inside + // samples, so every sync sample must have configOBUs re-injected to stay independently + // decodable/seekable. MP4Handle sample_h = nullptr; if (MP4NewHandle(0, &sample_h) != MP4NoErr || !sample_h) { spdlog::error("MP4NewHandle(sample) failed"); @@ -194,6 +213,7 @@ bool Av2Demuxer::demux(const std::string& input_mp4, const std::string& output_o } uint64_t total_sample_bytes = 0; + uint64_t total_config_bytes = 0; uint32_t samples_with_td = 0; for (uint32_t i = 1; i <= sample_count; ++i) { u32 size = 0; @@ -207,14 +227,30 @@ bool Av2Demuxer::demux(const std::string& input_mp4, const std::string& output_o MP4DisposeMovie(movie); return false; } + bool is_sync = !(flags & MP4MediaSampleNotSync); + const uint8_t* config_obus = nullptr; + size_t config_obus_size = 0; + if (is_sync) { + auto it = config_obus_map.find(desc_idx); + if (it == config_obus_map.end()) { + spdlog::error("Sample {} references desc_idx {} with no known av2C configOBUs", i, + desc_idx); + MP4DisposeHandle(sample_h); + MP4DisposeMovie(movie); + return false; + } + config_obus = it->second.data(); + config_obus_size = it->second.size(); + } + const uint8_t* sample_data = reinterpret_cast(*sample_h); size_t td_prefix = leading_td_length(sample_data, size); if (td_prefix > 0) { // Sample already carries its own TD. Emit the TD that came from the bitstream. - // For sample 1 splice configOBUs in between the TD and the frame data. + // For a sync sample splice configOBUs in between the TD and the frame data. out.write(reinterpret_cast(sample_data), static_cast(td_prefix)); - if (i == 1) { + if (is_sync) { out.write(reinterpret_cast(config_obus), static_cast(config_obus_size)); } @@ -224,7 +260,7 @@ bool Av2Demuxer::demux(const std::string& input_mp4, const std::string& output_o } else { // Sample has no TD (the default --drop-td muxing). Synthesize one. out.write(reinterpret_cast(kTdBytes), sizeof(kTdBytes)); - if (i == 1) { + if (is_sync) { out.write(reinterpret_cast(config_obus), static_cast(config_obus_size)); } @@ -232,15 +268,17 @@ bool Av2Demuxer::demux(const std::string& input_mp4, const std::string& output_o static_cast(size)); } total_sample_bytes += size; + total_config_bytes += config_obus_size; } MP4DisposeHandle(sample_h); MP4DisposeMovie(movie); out.close(); - spdlog::info("Wrote {} ({} samples [{} carried a TD, {} got a synthesized TD], {} configOBU bytes)", + spdlog::info("Wrote {} ({} samples [{} carried a TD, {} got a synthesized TD], {} sample " + "entries, {} configOBU bytes total)", output_obu, sample_count, samples_with_td, sample_count - samples_with_td, - config_obus_size); + config_obus_map.size(), total_config_bytes); return true; } diff --git a/apps/av2_mux/av2_codec_config.h b/apps/av2_mux/av2_codec_config.h index 4dd877b..ac1fa99 100644 --- a/apps/av2_mux/av2_codec_config.h +++ b/apps/av2_mux/av2_codec_config.h @@ -62,4 +62,11 @@ struct AV2CodecConfigurationBox { std::ifstream* input_file_ = nullptr; }; +inline bool operator==(const AV2CodecConfigurationBox& a, const AV2CodecConfigurationBox& b) { + return a.serialize() == b.serialize(); +} +inline bool operator!=(const AV2CodecConfigurationBox& a, const AV2CodecConfigurationBox& b) { + return !(a == b); +} + } // namespace av2_obu diff --git a/apps/av2_mux/av2_muxer.cpp b/apps/av2_mux/av2_muxer.cpp index 80157b5..148ad70 100644 --- a/apps/av2_mux/av2_muxer.cpp +++ b/apps/av2_mux/av2_muxer.cpp @@ -14,7 +14,6 @@ #include #include -#include #include #include @@ -80,7 +79,7 @@ bool Av2Muxer::package(const OBUParser& parser, const std::string& output_path) uint32_t sync_samples = 0; for (uint32_t i = strategy_.start_tu; i < end_tu; ++i) { int32_t ctts = need_ctts ? ctts_offsets[i - strategy_.start_tu] : 0; - if (!write_tu(tus[i], ctts)) { + if (!write_tu(tus[i], i, ctts)) { MUX_ERROR("Failed to write TU {}", i); return false; } @@ -103,15 +102,28 @@ bool Av2Muxer::package(const OBUParser& parser, const std::string& output_path) return true; } +namespace { + +std::vector read_obu_bytes(std::ifstream& ifs, const BaseOBU& o) { + const auto& pos = o.position(); + size_t total = pos.size_field_len + pos.header_len + pos.payload_size; + std::vector b(total); + ifs.clear(); + ifs.seekg(pos.start_pos); + ifs.read(reinterpret_cast(b.data()), static_cast(total)); + ifs.clear(); + return b; +} + +} // namespace + bool Av2Muxer::setup_video_track(const OBUParser& parser) { - const SequenceHeaderOBU* first_sh = find_first_sequence_header(parser); - if (!first_sh) { - MUX_ERROR("No Sequence Header OBU found"); + if (parser.temporal_units().empty()) { + MUX_ERROR("No temporal units found"); return false; } - const AV2SequenceHeader& sh = first_sh->sequence_header(); - // All extended layers of a temporal unit share the same timing info + // All extended layers of a temporal unit share the same timing info. // The base layer's frames can drive the CTS computation. base_xlayer_id_ = GLOBAL_XLAYER_ID; for (const auto& o : parser.obus()) { @@ -121,105 +133,114 @@ bool Av2Muxer::setup_video_track(const OBUParser& parser) { if (base_xlayer_id_ == GLOBAL_XLAYER_ID || x < base_xlayer_id_) base_xlayer_id_ = x; } - strategy_.any_non_monotonic = (sh.monotonic_output_order_flag == 0); - if (strategy_.any_non_monotonic) { - if (!check_doh_lifter_supported(parser)) return false; - doh_lifter_ = std::make_unique(sh.inter_config.OrderHintBits); - } + const auto& tus = parser.temporal_units(); + tu_desc_idx_.assign(tus.size(), 0); - AV2CodecConfigurationBox av2c(input_ifs_); - av2c.set_from_sequence_header(sh); + std::vector last_config_bytes; + uint32_t active_desc_idx = 0; - auto read_obu_bytes = [&](const BaseOBU& o) { - const auto& pos = o.position(); - size_t total = pos.size_field_len + pos.header_len + pos.payload_size; - std::vector b(total); - input_ifs_.clear(); - input_ifs_.seekg(pos.start_pos); - input_ifs_.read(reinterpret_cast(b.data()), static_cast(total)); - input_ifs_.clear(); - return b; - }; + for (uint32_t i = 0; i < tus.size(); ++i) { + if (!tus[i].is_sync_sample()) { + tu_desc_idx_[i] = active_desc_idx; + continue; + } - // Collect config OBUs in bitstream order from the first TU SHs, LCR, OPS, CI. - // If encoder interleaves a frame OBU between config OBUs, that frame simply lands sample - if (parser.temporal_units().empty()) { - MUX_ERROR("No temporal units found"); - return false; - } - std::map, std::vector> config_bytes; - size_t config_count = 0, sh_count = 0; - for (const BaseOBU* p : parser.temporal_units().front()) { - if (!is_config_obu(p->type())) continue; - config_bytes[{static_cast(p->type()), p->header().get_xlayer_id()}] = read_obu_bytes(*p); - if (p->type() == OBUType::SEQUENCE_HEADER) ++sh_count; - ++config_count; - if (!av2c.append_config_obu(*p)) { - MUX_ERROR("Failed to append config OBU (type {}) to av2C configOBUs", - static_cast(p->type())); + AV2CodecConfigurationBox av2c(input_ifs_); + std::vector config_bytes; + const SequenceHeaderOBU* sh = nullptr; + size_t config_count = 0; + + for (const BaseOBU* p : tus[i]) { + if (!is_config_obu(p->type())) continue; + if (p->type() == OBUType::SEQUENCE_HEADER && !sh) { + sh = dynamic_cast(p); + } + auto raw = read_obu_bytes(input_ifs_, *p); + config_bytes.insert(config_bytes.end(), raw.begin(), raw.end()); + ++config_count; + if (!av2c.append_config_obu(*p)) { + MUX_ERROR("Failed to append config OBU (type {}) to av2C configOBUs", + static_cast(p->type())); + return false; + } + } + + if (!sh) { + MUX_ERROR("Sync sample TU {} has no Sequence Header OBU", i); return false; } - } + const AV2SequenceHeader& sequence_header = sh->sequence_header(); + av2c.set_from_sequence_header(sequence_header); - // Config OBUs live only in the sample entry. If a later occurrence differs, - // we would need multiple SE which is not yet implemented. - for (const auto& obu_ptr : parser.obus()) { - const BaseOBU* p = obu_ptr.get(); - if (!is_config_obu(p->type())) continue; - auto it = config_bytes.find({static_cast(p->type()), p->header().get_xlayer_id()}); - if (it == config_bytes.end() || read_obu_bytes(*p) != it->second) { - config_changes_over_time_ = true; - break; + if (!config_bytes.empty() && config_bytes == last_config_bytes) { + tu_desc_idx_[i] = active_desc_idx; + continue; + } + last_config_bytes = config_bytes; + + std::optional colr = + strategy_.colr_override ? strategy_.colr_override : extract_colr_info_from_tu(tus[i]); + + uint32_t w_full = static_cast(sequence_header.max_frame_width_minus_1) + 1; + uint32_t h_full = static_cast(sequence_header.max_frame_height_minus_1) + 1; + if (w_full > 0xFFFF || h_full > 0xFFFF) { + MUX_WARN("Frame dimensions {}x{} exceed 16-bit av2C field; truncating", w_full, h_full); + } + uint16_t width = static_cast(w_full & 0xFFFF); + uint16_t height = static_cast(h_full & 0xFFFF); + + uint32_t new_desc_idx; + if (sample_entries_.empty()) { + strategy_.any_non_monotonic = (sequence_header.monotonic_output_order_flag == 0); + if (strategy_.any_non_monotonic) { + if (!check_doh_lifter_supported(parser)) return false; + doh_lifter_ = + std::make_unique(sequence_header.inter_config.OrderHintBits); + } + + if (!writer_.add_video_track(strategy_.timescale, width, height, av2c)) return false; + writer_.set_samples_per_chunk(strategy_.samples_per_chunk); + if (colr && !writer_.add_colr_nclx(*colr)) return false; + new_desc_idx = 1; + } else { + new_desc_idx = writer_.add_sample_entry(av2c, width, height, colr); + if (new_desc_idx == 0) return false; + MUX_INFO("CVS boundary at TU {}: new sample entry (desc_idx={})", i, new_desc_idx); } - } - if (config_changes_over_time_) { - MUX_WARN( - "Configuration OBUs change over the course of the track. Multi-sample-entry support is not yet " - "implemented; only the first temporal unit's configuration is carried in av2C. Samples that " - "depend on later configuration may decode incorrectly."); + MUX_DEBUG("configOBUs (desc_idx={}): {} OBU(s)", new_desc_idx, config_count); + if (colr) { + MUX_DEBUG("colr/nclx ({}): cp={} tc={} mc={} full_range={}", + strategy_.colr_override ? "override" : "extracted from bitstream", + colr->colour_primaries, colr->transfer_characteristics, colr->matrix_coefficients, + colr->full_range_flag); + } else { + MUX_DEBUG("No CI/LCR/OPS color info found and no --colr-override; omitting colr box"); + } + last_colr_ = colr; + + sample_entries_.push_back({av2c, colr, new_desc_idx, config_bytes}); + active_desc_idx = new_desc_idx; + tu_desc_idx_[i] = active_desc_idx; } - MUX_DEBUG("configOBUs: {} OBU(s) ({} sequence header(s))", config_count, sh_count); - uint32_t w_full = static_cast(sh.max_frame_width_minus_1) + 1; - uint32_t h_full = static_cast(sh.max_frame_height_minus_1) + 1; - if (w_full > 0xFFFF || h_full > 0xFFFF) { - MUX_WARN("Frame dimensions {}x{} exceed 16-bit av2C field; truncating", w_full, h_full); + if (sample_entries_.empty()) { + MUX_ERROR("No sync sample found in stream"); + return false; } - uint16_t width = static_cast(w_full & 0xFFFF); - uint16_t height = static_cast(h_full & 0xFFFF); - - if (!writer_.add_video_track(strategy_.timescale, width, height, av2c)) return false; - writer_.set_samples_per_chunk(strategy_.samples_per_chunk); - - std::optional colr = - strategy_.colr_override ? strategy_.colr_override : extract_colr_info(parser); - if (colr) { - if (!writer_.add_colr_nclx(*colr)) return false; - MUX_DEBUG("colr/nclx ({}): cp={} tc={} mc={} full_range={}", - strategy_.colr_override ? "override" : "extracted from bitstream", - colr->colour_primaries, colr->transfer_characteristics, colr->matrix_coefficients, - colr->full_range_flag); - } else { - MUX_DEBUG("No CI/LCR/OPS color info found and no --colr-override; omitting colr box"); + + if (sample_entries_.size() > 1) { + MUX_DEBUG("Multi-CVS stream: {} sample entries", sample_entries_.size()); + check_cmaf_invariants(sample_entries_); } - last_colr_ = colr; return true; } bool Av2Muxer::check_doh_lifter_supported(const OBUParser& parser) { - // For multi-layer streams the lifter runs on the base extended layer only - uint32_t clk_count = 0; - for (const auto& obu : parser.obus()) { - if (obu->header().get_xlayer_id() != base_xlayer_id_) continue; - if (obu->type() == OBUType::CLK) ++clk_count; - } - if (clk_count > 1) { - MUX_ERROR("Non-monotonic stream: base xlayer has {} CLK frames (multi-CVS); the v1 DOH " - "lifter does not reset across CVS boundaries.", clk_count); - return false; - } + // For multi-layer streams the lifter runs on the base extended layer only. + // Multi-CVS non-monotonic streams are supported: the lifter is reset at each + // CVS boundary (see compute_composition_offsets()). for (const auto& obu : parser.obus()) { if (obu->header().get_xlayer_id() != base_xlayer_id_) continue; if (obu->type() == OBUType::BRIDGE_FRAME) { @@ -242,11 +263,12 @@ bool Av2Muxer::check_doh_lifter_supported(const OBUParser& parser) { return true; } -bool Av2Muxer::write_tu(const TemporalUnit& tu, int32_t composition_offset) { +bool Av2Muxer::write_tu(const TemporalUnit& tu, uint32_t tu_index, int32_t composition_offset) { std::vector bytes; if (!assemble_sample_bytes(tu, bytes)) return false; + uint32_t desc_idx = tu_desc_idx_[tu_index]; return writer_.add_sample(bytes, strategy_.default_sample_duration, tu.is_sync_sample(), - composition_offset); + composition_offset, desc_idx); } std::vector Av2Muxer::compute_composition_offsets( @@ -256,18 +278,27 @@ std::vector Av2Muxer::compute_composition_offsets( offsets.reserve(end - start); const int64_t dur = static_cast(strategy_.default_sample_duration); const SequenceHeaderOBU* first_sh = find_first_sequence_header(parser); - const AV2SequenceHeader& sh = first_sh->sequence_header(); + const AV2SequenceHeader* sh = &first_sh->sequence_header(); for (uint32_t i = start; i < end; ++i) { int32_t off = 0; int64_t output_doh = -1; + if (tus[i].is_sync_sample() && i > start) { + if (const SequenceHeaderOBU* cvs_sh = find_sh_in_tu(tus[i], base_xlayer_id_)) { + doh_lifter_ = + std::make_unique(cvs_sh->sequence_header().inter_config.OrderHintBits); + ref_buffer_ = RefFrameBuffer{}; + sh = &cvs_sh->sequence_header(); + } + } + // Feed only the base extended layer's frames to the lifter (sample has shared timing for all layers) for (const auto* obu : tus[i].obus()) { if (obu->header().get_xlayer_id() != base_xlayer_id_) continue; const FrameHeaderInfo* fh = frame_header_of(obu); if (!fh) continue; - const int64_t doh = doh_lifter_->process(*obu, *fh, sh, ref_buffer_); + const int64_t doh = doh_lifter_->process(*obu, *fh, *sh, ref_buffer_); if (fh->is_output_frame) output_doh = doh; } @@ -312,6 +343,57 @@ const SequenceHeaderOBU* Av2Muxer::find_first_sequence_header(const OBUParser& p return nullptr; } +const SequenceHeaderOBU* Av2Muxer::find_sh_in_tu(const TemporalUnit& tu, uint32_t xlayer_id) { + for (const BaseOBU* obu : tu.obus()) { + if (obu->type() != OBUType::SEQUENCE_HEADER) continue; + if (obu->header().get_xlayer_id() != xlayer_id) continue; + if (auto* sh = dynamic_cast(obu)) return sh; + } + return nullptr; +} + +void Av2Muxer::check_cmaf_invariants(const std::vector& entries) { + const auto& first = entries.front().av2c; + bool color_warned = false; + for (size_t i = 1; i < entries.size(); ++i) { + const auto& e = entries[i].av2c; + if (e.seq_profile_idc != first.seq_profile_idc) { + MUX_WARN("Sample entry {} has seq_profile_idc={} (first={}); players may not support " + "switching profiles mid-stream.", + i, e.seq_profile_idc, first.seq_profile_idc); + } + if (e.still_picture != first.still_picture) { + MUX_WARN("Sample entry {} has still_picture={} (first={})", i, e.still_picture, + first.still_picture); + } + if (e.seq_level_idx != first.seq_level_idx) { + MUX_WARN("Sample entry {} has seq_level_idx={} (first={})", i, e.seq_level_idx, + first.seq_level_idx); + } + if (e.seq_tier != first.seq_tier) { + MUX_WARN("Sample entry {} has seq_tier={} (first={})", i, e.seq_tier, first.seq_tier); + } + if (e.seq_initial_display_delay_minus_1 != first.seq_initial_display_delay_minus_1) { + MUX_WARN("Sample entry {} has seq_initial_display_delay_minus_1={} (first={})", i, + e.seq_initial_display_delay_minus_1, first.seq_initial_display_delay_minus_1); + } + if (!color_warned && entries[i].colr.has_value() != entries.front().colr.has_value()) { + MUX_WARN("Sample entry {} colr presence differs from first sample entry", i); + color_warned = true; + } else if (!color_warned && entries[i].colr && entries.front().colr) { + const auto& c0 = *entries.front().colr; + const auto& ci = *entries[i].colr; + if (c0.colour_primaries != ci.colour_primaries || + c0.transfer_characteristics != ci.transfer_characteristics || + c0.matrix_coefficients != ci.matrix_coefficients || + c0.full_range_flag != ci.full_range_flag) { + MUX_WARN("Sample entry {} colr/nclx differs from first sample entry", i); + color_warned = true; + } + } + } +} + void log_stream_summary(const OBUParser& parser) { const auto stats = parser.get_statistics(); MUX_DEBUG("Stream summary:"); diff --git a/apps/av2_mux/av2_muxer.h b/apps/av2_mux/av2_muxer.h index cf5635b..daba696 100644 --- a/apps/av2_mux/av2_muxer.h +++ b/apps/av2_mux/av2_muxer.h @@ -11,12 +11,14 @@ #pragma once +#include #include #include #include #include #include +#include "av2_codec_config.h" #include "colr_info.h" #include "display_order_lifter.h" #include "mp4_writer.h" @@ -30,6 +32,14 @@ class TemporalUnit; class SequenceHeaderOBU; class BaseOBU; +// One entry per distinct av2C configuration (CVS) observed in the stream. +struct SampleEntryRecord { + AV2CodecConfigurationBox av2c; + std::optional colr; + uint32_t desc_idx = 0; // 1-based index assigned by Mp4Writer + std::vector config_obu_bytes; // serialized, for equality checks +}; + // Drives mux of one elementary AV2 bitstream into one ISOBMFF .mp4. class Av2Muxer { public: @@ -40,13 +50,15 @@ class Av2Muxer { private: bool setup_video_track(const OBUParser& parser); bool check_doh_lifter_supported(const OBUParser& parser); - bool write_tu(const TemporalUnit& tu, int32_t composition_offset); + bool write_tu(const TemporalUnit& tu, uint32_t tu_index, int32_t composition_offset); bool assemble_sample_bytes(const TemporalUnit& tu, std::vector& out); std::vector compute_composition_offsets(const OBUParser& parser, const std::vector& tus, uint32_t start, uint32_t end); static const SequenceHeaderOBU* find_first_sequence_header(const OBUParser& parser); + static const SequenceHeaderOBU* find_sh_in_tu(const TemporalUnit& tu, uint32_t xlayer_id); + static void check_cmaf_invariants(const std::vector& entries); std::string input_path_; MuxStrategy strategy_; @@ -56,8 +68,11 @@ class Av2Muxer { RefFrameBuffer ref_buffer_; std::optional last_colr_; - // Detect (SH, LCR, OPS, CI) changes over time -> would require multiple SE - bool config_changes_over_time_ = false; + // One entry per distinct av2C configuration (CVS) registered with the writer. + std::vector sample_entries_; + + // Maps TU index -> desc_idx, populated during setup_video_track(). + std::vector tu_desc_idx_; // Lowest non-global obu_xlayer_id among frame OBUs (all layers share timing info) uint32_t base_xlayer_id_ = 0; diff --git a/apps/av2_mux/colr_info.cpp b/apps/av2_mux/colr_info.cpp index f7480c6..47aa120 100644 --- a/apps/av2_mux/colr_info.cpp +++ b/apps/av2_mux/colr_info.cpp @@ -13,9 +13,11 @@ #include #include +#include #include #include +#include #include #include #include @@ -24,6 +26,11 @@ namespace av2_obu { namespace { +// Normalizes both parser.obus() (unique_ptr) and TemporalUnit::obus() +// (raw const BaseOBU*) to a raw pointer for shared iteration logic below. +const BaseOBU* as_raw(const std::unique_ptr& obu) { return obu.get(); } +const BaseOBU* as_raw(const BaseOBU* obu) { return obu; } + // We can collect a few well-known CICP presets here const std::map& named_colr_profiles() { static const std::map kProfiles = { @@ -39,10 +46,12 @@ ColrInfo make(uint32_t cp, uint32_t tc, uint32_t mc, uint32_t fr) { return ColrInfo{cp, tc, mc, fr}; } -std::optional from_ci(const OBUParser& parser) { - for (const auto& obu : parser.obus()) { +template +std::optional from_ci(const Range& obus) { + for (const auto& item : obus) { + const BaseOBU* obu = as_raw(item); if (obu->type() != OBUType::CONTENT_INTERPRETATION) continue; - auto* ci = dynamic_cast(obu.get()); + auto* ci = dynamic_cast(obu); if (!ci || !ci->has_color_description()) continue; return make(ci->color_primaries(), ci->transfer_characteristics(), ci->matrix_coefficients(), ci->full_range_flag()); @@ -50,12 +59,14 @@ std::optional from_ci(const OBUParser& parser) { return std::nullopt; } -std::optional from_lcr(const OBUParser& parser) { +template +std::optional from_lcr(const Range& obus) { std::optional best; uint32_t best_xid = std::numeric_limits::max(); - for (const auto& obu : parser.obus()) { + for (const auto& item : obus) { + const BaseOBU* obu = as_raw(item); if (obu->type() != OBUType::LAYER_CONFIGURATION_RECORD) continue; - auto* lcr = dynamic_cast(obu.get()); + auto* lcr = dynamic_cast(obu); if (!lcr) continue; const auto& gp = lcr->global_payloads(); const auto& ids = lcr->xlayer_ids(); @@ -77,10 +88,12 @@ std::optional from_lcr(const OBUParser& parser) { return best; } -std::optional from_ops(const OBUParser& parser) { - for (const auto& obu : parser.obus()) { +template +std::optional from_ops(const Range& obus) { + for (const auto& item : obus) { + const BaseOBU* obu = as_raw(item); if (obu->type() != OBUType::OPERATING_POINT_SET) continue; - auto* ops = dynamic_cast(obu.get()); + auto* ops = dynamic_cast(obu); if (!ops) continue; for (const auto& op : ops->operating_points()) { if (!op.has_color_info) continue; @@ -91,13 +104,22 @@ std::optional from_ops(const OBUParser& parser) { return std::nullopt; } +template +std::optional extract_colr_info_impl(const Range& obus) { + if (auto v = from_ci(obus)) return v; + if (auto v = from_lcr(obus)) return v; + if (auto v = from_ops(obus)) return v; + return std::nullopt; +} + } // namespace std::optional extract_colr_info(const OBUParser& parser) { - if (auto v = from_ci(parser)) return v; - if (auto v = from_lcr(parser)) return v; - if (auto v = from_ops(parser)) return v; - return std::nullopt; + return extract_colr_info_impl(parser.obus()); +} + +std::optional extract_colr_info_from_tu(const TemporalUnit& tu) { + return extract_colr_info_impl(tu.obus()); } std::optional colr_profile_by_name(const std::string& name) { diff --git a/apps/av2_mux/colr_info.h b/apps/av2_mux/colr_info.h index 1ca8dfe..45c219b 100644 --- a/apps/av2_mux/colr_info.h +++ b/apps/av2_mux/colr_info.h @@ -18,6 +18,7 @@ namespace av2_obu { class OBUParser; +class TemporalUnit; // CICP color metadata for the ISOBMFF 'colr' / nclx box. struct ColrInfo { @@ -31,6 +32,10 @@ struct ColrInfo { // Returns nullopt if no source carries color metadata. std::optional extract_colr_info(const OBUParser& parser); +// Same search order as extract_colr_info(), scoped to a single TU's OBUs. +// Used to pick colr for a new sample entry created at a CVS boundary. +std::optional extract_colr_info_from_tu(const TemporalUnit& tu); + // Resolves a CICP profile name (e.g. "bt709") to its ColrInfo. nullopt for unknown names std::optional colr_profile_by_name(const std::string& name); diff --git a/apps/av2_mux/mp4_writer.cpp b/apps/av2_mux/mp4_writer.cpp index ce50acb..7a26503 100644 --- a/apps/av2_mux/mp4_writer.cpp +++ b/apps/av2_mux/mp4_writer.cpp @@ -55,7 +55,9 @@ Mp4Writer::Mp4Writer() { } Mp4Writer::~Mp4Writer() { - if (sample_entry_) MP4DisposeHandle(sample_entry_); + for (auto& [idx, handle] : sample_entries_) { + if (handle) MP4DisposeHandle(handle); + } if (movie_) MP4DisposeMovie(movie_); } @@ -75,51 +77,11 @@ bool Mp4Writer::add_video_track(uint32_t timescale, uint16_t width, uint16_t hei return false; } - std::vector av2c_bytes = av2c.serialize(); - if (av2c_bytes.empty()) { - MUX_ERROR("av2C serialization produced empty payload"); - return false; - } - - MP4Handle av2c_payload = make_handle(av2c_bytes.data(), av2c_bytes.size()); - if (!av2c_payload) { - MUX_ERROR("MP4NewHandle(av2C payload) failed"); - return false; - } - - MP4GenericAtom av2c_atom = nullptr; - err = MP4NewForeignAtom(&av2c_atom, kAtomTypeAv2C, av2c_payload); - if (err != MP4NoErr || !av2c_atom) { - MUX_ERROR("MP4NewForeignAtom('av2C') failed (err={})", err); - MP4DisposeHandle(av2c_payload); - return false; - } - - err = MP4NewHandle(0, &sample_entry_); - if (err != MP4NoErr || !sample_entry_) { - MUX_ERROR("MP4NewHandle(sample_entry) failed (err={})", err); - return false; - } - - err = ISONewGeneralSampleDescription(track_, sample_entry_, /*dataRefIdx=*/1, - kSampleEntryTypeAv02, av2c_atom); - if (err != MP4NoErr) { - MUX_ERROR("ISONewGeneralSampleDescription failed (err={})", err); - return false; - } - - err = ISOSetSampleDescriptionDimensions(sample_entry_, width, height); - if (err != MP4NoErr) { - MUX_ERROR("ISOSetSampleDescriptionDimensions failed (err={})", err); - return false; - } - - return true; + uint32_t desc_idx = add_sample_entry(av2c, width, height, std::nullopt); + return desc_idx == 1; } -bool Mp4Writer::add_colr_nclx(const ColrInfo& info) { - if (!sample_entry_) return false; - +bool Mp4Writer::attach_colr_nclx(MP4Handle entry, const ColrInfo& info) { // TODO(https://github.com/MPEGGroup/isobmff/issues/76): replace with a libisomedia helper once landed. // ColourInformationBox uint8_t payload[11]; @@ -149,7 +111,7 @@ bool Mp4Writer::add_colr_nclx(const ColrInfo& info) { return false; } - err = ISOAddAtomToSampleDescription(sample_entry_, atom); + err = ISOAddAtomToSampleDescription(entry, atom); if (err != MP4NoErr) { MUX_ERROR("ISOAddAtomToSampleDescription('colr') failed (err={})", err); return false; @@ -161,9 +123,77 @@ bool Mp4Writer::add_colr_nclx(const ColrInfo& info) { return true; } +bool Mp4Writer::add_colr_nclx(const ColrInfo& info) { + auto it = sample_entries_.find(1); + if (it == sample_entries_.end()) return false; + return attach_colr_nclx(it->second, info); +} + +uint32_t Mp4Writer::add_sample_entry(const AV2CodecConfigurationBox& av2c, uint16_t width, + uint16_t height, const std::optional& colr) { + if (!track_) return 0; + + std::vector av2c_bytes = av2c.serialize(); + if (av2c_bytes.empty()) { + MUX_ERROR("av2C serialization produced empty payload"); + return 0; + } + + MP4Handle av2c_payload = make_handle(av2c_bytes.data(), av2c_bytes.size()); + if (!av2c_payload) { + MUX_ERROR("MP4NewHandle(av2C payload) failed"); + return 0; + } + + MP4GenericAtom av2c_atom = nullptr; + MP4Err err = MP4NewForeignAtom(&av2c_atom, kAtomTypeAv2C, av2c_payload); + if (err != MP4NoErr || !av2c_atom) { + MUX_ERROR("MP4NewForeignAtom('av2C') failed (err={})", err); + MP4DisposeHandle(av2c_payload); + return 0; + } + + MP4Handle entry = nullptr; + err = MP4NewHandle(0, &entry); + if (err != MP4NoErr || !entry) { + MUX_ERROR("MP4NewHandle(sample_entry) failed (err={})", err); + return 0; + } + + err = ISONewGeneralSampleDescription(track_, entry, /*dataRefIdx=*/1, kSampleEntryTypeAv02, + av2c_atom); + if (err != MP4NoErr) { + MUX_ERROR("ISONewGeneralSampleDescription failed (err={})", err); + MP4DisposeHandle(entry); + return 0; + } + + err = ISOSetSampleDescriptionDimensions(entry, width, height); + if (err != MP4NoErr) { + MUX_ERROR("ISOSetSampleDescriptionDimensions failed (err={})", err); + MP4DisposeHandle(entry); + return 0; + } + + if (colr && !attach_colr_nclx(entry, *colr)) { + MP4DisposeHandle(entry); + return 0; + } + + uint32_t desc_idx = ++next_desc_idx_; + sample_entries_[desc_idx] = entry; + return desc_idx; +} + bool Mp4Writer::add_sample(const std::vector& bytes, uint32_t duration, bool is_sync, - int32_t composition_offset) { - if (!media_ || !sample_entry_) return false; + int32_t composition_offset, uint32_t desc_idx) { + if (!media_ || sample_entries_.find(desc_idx) == sample_entries_.end()) return false; + + if (current_desc_idx_ != 0 && desc_idx != current_desc_idx_) { + if (!flush_chunk()) return false; + } + current_desc_idx_ = desc_idx; + pending_sizes_.push_back(static_cast(bytes.size())); pending_data_.insert(pending_data_.end(), bytes.begin(), bytes.end()); pending_durations_.push_back(duration); @@ -211,7 +241,8 @@ bool Mp4Writer::flush_chunk() { ctts = make_handle(pending_ctts_offsets_.data(), n * sizeof(int32_t)); } - MP4Handle entry_for_call = first_chunk_ ? sample_entry_ : nullptr; + bool already_attached = entries_attached_.count(current_desc_idx_) > 0; + MP4Handle entry_for_call = already_attached ? nullptr : sample_entries_.at(current_desc_idx_); MP4Err err = MP4AddMediaSamples(media_, data, n, durations, sizes, entry_for_call, /*decodingOffsetsH=*/ctts, sync); @@ -227,7 +258,7 @@ bool Mp4Writer::flush_chunk() { return false; } - first_chunk_ = false; + entries_attached_.insert(current_desc_idx_); pending_data_.clear(); pending_sizes_.clear(); pending_durations_.clear(); diff --git a/apps/av2_mux/mp4_writer.h b/apps/av2_mux/mp4_writer.h index 48a4cbd..ba5a40e 100644 --- a/apps/av2_mux/mp4_writer.h +++ b/apps/av2_mux/mp4_writer.h @@ -12,6 +12,9 @@ #pragma once #include +#include +#include +#include #include #include @@ -23,7 +26,9 @@ namespace av2_obu { -// Codec-agnostic libisomedia wrapper for a single AV2 video track. +// Codec-agnostic libisomedia wrapper for a single AV2 video track. Supports +// multiple sample entries (stsd) on the track, one per CVS whose configOBUs +// differ from the previous CVS. class Mp4Writer { public: Mp4Writer(); @@ -32,20 +37,30 @@ class Mp4Writer { Mp4Writer(const Mp4Writer&) = delete; Mp4Writer& operator=(const Mp4Writer&) = delete; + // Creates the track and its first sample entry (desc_idx == 1). bool add_video_track(uint32_t timescale, uint16_t width, uint16_t height, const AV2CodecConfigurationBox& av2c); // Must be called after add_video_track() and before the first add_sample(). + // Attaches colr to the first sample entry (desc_idx == 1). bool add_colr_nclx(const ColrInfo& info); + // Creates an additional sample entry on the same track (CVS boundary with + // changed configOBUs). Returns the assigned desc_idx (>= 2), or 0 on failure. + uint32_t add_sample_entry(const AV2CodecConfigurationBox& av2c, uint16_t width, + uint16_t height, const std::optional& colr); + // Must be called before any add_sample() bool enable_signed_composition_offsets(); // 0 disables auto-flush; caller drives chunk boundaries via flush_chunk()/finalize(). void set_samples_per_chunk(uint32_t n) { samples_per_chunk_ = n; } + // desc_idx selects which sample entry (from add_video_track()/add_sample_entry()) + // this sample belongs to. Changing desc_idx from the previous call forces a + // chunk flush first, since a chunk cannot span sample entries. bool add_sample(const std::vector& bytes, uint32_t duration, bool is_sync, - int32_t composition_offset = 0); + int32_t composition_offset, uint32_t desc_idx); // One flush_chunk() = one MP4AddMediaSamples() call = one stsc chunk. bool flush_chunk(); @@ -56,10 +71,19 @@ class Mp4Writer { MP4Movie movie_ = nullptr; MP4Track track_ = nullptr; MP4Media media_ = nullptr; - MP4Handle sample_entry_ = nullptr; - // First chunk supplies sample_entry_; later chunks pass NULL to reuse it. - bool first_chunk_ = true; + // Sample entries by desc_idx (1-based, matching stsd entry order). + std::map sample_entries_; + uint32_t next_desc_idx_ = 0; + + // desc_idx of the samples currently buffered in pending_*. + uint32_t current_desc_idx_ = 0; + + // desc_idx values already passed to MP4AddMediaSamples as sampleEntryH. + // libisomedia appends a brand-new stsd entry every time a non-null + // sampleEntryH is passed, so each entry may be attached only once; later + // chunks reusing that entry must pass NULL to keep it "current". + std::set entries_attached_; std::vector pending_data_; std::vector pending_sizes_; @@ -69,6 +93,8 @@ class Mp4Writer { bool signed_ctts_enabled_ = false; uint32_t samples_per_chunk_ = 0; + + bool attach_colr_nclx(MP4Handle entry, const ColrInfo& info); }; } // namespace av2_obu