From b2cb1061e4dff7c8f263947f0320e79bc30bdd91 Mon Sep 17 00:00:00 2001 From: Praveen <16594062+14praveenk@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:02:35 +0100 Subject: [PATCH 1/2] Improve video decoder resync handling --- .../cpp/libaasdk-jni/messenger/Timestamp.cpp | 16 ++-- .../smg/hu/projection/NativeVideoOutput.java | 18 ++--- .../cpp/libomxvideocodec-jni/H264NalParser.h | 71 +++++++++++++++++ .../cpp/libomxvideocodec-jni/OMXDecoder.cpp | 8 +- .../cpp/libomxvideocodec-jni/OMXSource.cpp | 77 ++++++++++++++++--- .../main/cpp/libomxvideocodec-jni/OMXSource.h | 4 + 6 files changed, 156 insertions(+), 38 deletions(-) create mode 100644 omxvideocodec/src/main/cpp/libomxvideocodec-jni/H264NalParser.h diff --git a/aasdk/src/main/cpp/libaasdk-jni/messenger/Timestamp.cpp b/aasdk/src/main/cpp/libaasdk-jni/messenger/Timestamp.cpp index d66c2268..048bfdf1 100644 --- a/aasdk/src/main/cpp/libaasdk-jni/messenger/Timestamp.cpp +++ b/aasdk/src/main/cpp/libaasdk-jni/messenger/Timestamp.cpp @@ -1,7 +1,6 @@ #include #include -#include -#include +#include namespace aasdk { @@ -15,12 +14,15 @@ Timestamp::Timestamp(ValueType stamp) } Timestamp::Timestamp(const common::DataConstBuffer& buffer) + : stamp_(0) { -// const ValueType& timestampBig = reinterpret_cast(buffer.cdata[0]); // generates sigbus on release build -// auto timestampBig = reinterpret_cast(&buffer.cdata[0]); // result in wrong timestamp -// stamp_ = boost::endian::big_to_native(timestampBig); - stamp_ = std::chrono::system_clock::now().time_since_epoch().count(); - Log_v("stamp_ %lld", stamp_); + if (buffer.size < sizeof(ValueType)) { + return; + } + + ValueType timestampBig; + std::memcpy(×tampBig, buffer.cdata, sizeof(timestampBig)); + stamp_ = boost::endian::big_to_native(timestampBig); } common::Data Timestamp::getData() const diff --git a/app/src/main/java/it/smg/hu/projection/NativeVideoOutput.java b/app/src/main/java/it/smg/hu/projection/NativeVideoOutput.java index 7b9dd36b..9c13d286 100644 --- a/app/src/main/java/it/smg/hu/projection/NativeVideoOutput.java +++ b/app/src/main/java/it/smg/hu/projection/NativeVideoOutput.java @@ -20,8 +20,8 @@ public class NativeVideoOutput extends VideoOutput implements Runnable { private static final String TAG = "NativeVideoOutput"; private MediaCodec codec_; - private boolean configured_; - private boolean running_; + private volatile boolean configured_; + private volatile boolean running_; private Thread codecThread_; public NativeVideoOutput(SurfaceView surfaceView){ @@ -57,10 +57,10 @@ public boolean init() { // format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 655360); codec_.configure(format, surface, null, 0); - configured_ = true; codec_.start(); - codecThread_.start(); + configured_ = true; running_ = true; + codecThread_.start(); return true; } catch (IOException e) { return false; @@ -113,17 +113,9 @@ public void run() { MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); while (running_) { if (configured_) { - int index = codec_.dequeueOutputBuffer(info, 0); + int index = codec_.dequeueOutputBuffer(info, 10000); if (index >= 0) { if (Log.isVerbose()) Log.v(TAG, "outputBufferIndex: " + index); - ByteBuffer buffer = null; - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - buffer = codec_.getOutputBuffer(index); - } else { - buffer = codec_.getOutputBuffers()[index]; - } - if (Log.isVerbose()) Log.v(TAG, "outputBuffer: " + buffer); - // setting true is telling system to render frame onto Surface codec_.releaseOutputBuffer(index, true); } diff --git a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/H264NalParser.h b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/H264NalParser.h new file mode 100644 index 00000000..a984dae9 --- /dev/null +++ b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/H264NalParser.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include + +namespace h264 { + +struct NalInfo { + bool hasCodecConfig = false; + bool hasIdr = false; +}; + +inline void addNalType(NalInfo& info, uint8_t header) { + const uint8_t type = header & 0x1f; + info.hasCodecConfig = info.hasCodecConfig || type == 7 || type == 8; + info.hasIdr = info.hasIdr || type == 5; +} + +inline NalInfo inspectNalUnits(const uint8_t* data, std::size_t size) { + NalInfo info; + if (data == nullptr || size == 0) { + return info; + } + + bool foundAnnexB = false; + + for (std::size_t i = 0; i + 3 < size; ++i) { + std::size_t headerOffset = 0; + if (data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1) { + headerOffset = i + 3; + } else if (i + 4 < size && data[i] == 0 && data[i + 1] == 0 && + data[i + 2] == 0 && data[i + 3] == 1) { + headerOffset = i + 4; + } + + if (headerOffset < size && headerOffset != 0) { + addNalType(info, data[headerOffset]); + foundAnnexB = true; + i = headerOffset; + } + } + + if (foundAnnexB) { + return info; + } + + // Android Auto normally sends Annex B, but accept four-byte AVCC lengths too. + std::size_t offset = 0; + bool foundAvcc = false; + while (offset + 4 < size) { + const uint32_t nalSize = (static_cast(data[offset]) << 24) | + (static_cast(data[offset + 1]) << 16) | + (static_cast(data[offset + 2]) << 8) | + static_cast(data[offset + 3]); + if (nalSize == 0 || nalSize > size - offset - 4) { + break; + } + + addNalType(info, data[offset + 4]); + foundAvcc = true; + offset += 4 + nalSize; + } + + if (!foundAvcc && size > 0) { + addNalType(info, data[0]); + } + + return info; +} + +} // namespace h264 diff --git a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXDecoder.cpp b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXDecoder.cpp index 754520df..464be229 100644 --- a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXDecoder.cpp +++ b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXDecoder.cpp @@ -71,7 +71,7 @@ OMXDecoder::~OMXDecoder(){ status_t OMXDecoder::read(){ if (Log::isVerbose()) Log_v("read"); - MediaBuffer *videoBuffer; + MediaBuffer *videoBuffer = nullptr; MediaSource::ReadOptions options; options.setLateBy(0); status_t ret = decoder_->read(&videoBuffer, &options); @@ -81,10 +81,8 @@ status_t OMXDecoder::read(){ if (videoBuffer->range_length() > 0) { if (Log::isVerbose()) Log_v("videobuffer length %d", videoBuffer->range_length()); sp metaData = videoBuffer->meta_data(); - int64_t timeUs = 0; - metaData->findInt64(kKeyTime, &timeUs); native_window_set_buffers_timestamp(nativeWindow_.get(), - timeUs); //NATIVE_WINDOW_TIMESTAMP_AUTO + NATIVE_WINDOW_TIMESTAMP_AUTO); ret = nativeWindow_->queueBuffer(nativeWindow_.get(), videoBuffer->graphicBuffer().get()); if (ret == 0) { @@ -103,5 +101,3 @@ status_t OMXDecoder::read(){ return ret; } - - diff --git a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp index 2b2b3c6f..1f6ac84c 100644 --- a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp +++ b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp @@ -1,4 +1,5 @@ #include "OMXSource.h" +#include "H264NalParser.h" #include "Log.h" #include @@ -9,11 +10,18 @@ using namespace android; +namespace { + +constexpr std::size_t kMaxQueuedBuffers = 8; +constexpr int64_t kMaxQueuedDurationUs = 250000; + +} // namespace + // https://www.programmersought.com/article/87712558400/ // https://vec.io/posts/use-android-hardware-decoder-with-omxcodec-in-ndk // https://stackoverflow.com/questions/9832503/android-include-native-stagefright-features-in-my-own-project OMXSource::OMXSource(int width, int height, int fps): - format_(nullptr), quitFlag_(false) + format_(nullptr), quitFlag_(false), waitingForSync_(false), droppedBuffers_(0) { int32_t bufferSize = (width * height * 3) / 2; @@ -41,11 +49,55 @@ void OMXSource::queueBuffer(MediaBuffer* buffer){ if (Log::isVerbose()) Log_v("add buffer to queue"); std::unique_lock l(mutex_); - if (!quitFlag_) { - pbuffers_.push(buffer); - if (Log::isVerbose()) Log_v("queueBuffer new size %d", pbuffers_.size()); - cond_.notify_one(); + if (quitFlag_) { + buffer->release(); + return; + } + + const auto* bufferData = static_cast(buffer->data()); + const auto* data = bufferData == nullptr ? nullptr : bufferData + buffer->range_offset(); + const h264::NalInfo nalInfo = h264::inspectNalUnits(data, buffer->range_length()); + int64_t timestampUs = 0; + int64_t oldestTimestampUs = 0; + buffer->meta_data()->findInt64(kKeyTime, ×tampUs); + + bool durationExceeded = false; + if (!pbuffers_.empty() && timestampUs > 0 && + pbuffers_.front()->meta_data()->findInt64(kKeyTime, &oldestTimestampUs) && + oldestTimestampUs > 0 && timestampUs > oldestTimestampUs) { + durationExceeded = timestampUs - oldestTimestampUs > kMaxQueuedDurationUs; + } + + if (pbuffers_.size() >= kMaxQueuedBuffers || durationExceeded) { + droppedBuffers_ += pbuffers_.size(); + clearQueuedBuffers(); + waitingForSync_ = true; + if (Log::isWarn()) { + Log_w("video decoder backlog exceeded %lld us; dropping until the next IDR frame", + static_cast(kMaxQueuedDurationUs)); + } + } + + if (waitingForSync_ && !nalInfo.hasCodecConfig && !nalInfo.hasIdr) { + ++droppedBuffers_; + buffer->release(); + return; + } + + buffer->meta_data()->setInt32(kKeyIsSyncFrame, nalInfo.hasIdr ? 1 : 0); + pbuffers_.push(buffer); + + if (nalInfo.hasIdr && waitingForSync_) { + waitingForSync_ = false; + if (Log::isWarn()) { + Log_w("video decoder resynchronized after dropping %d buffers", + static_cast(droppedBuffers_)); + } + droppedBuffers_ = 0; } + + if (Log::isVerbose()) Log_v("queueBuffer new size %d", pbuffers_.size()); + cond_.notify_one(); } sp OMXSource::getFormat(){ @@ -82,7 +134,6 @@ status_t OMXSource::read(MediaBuffer **buffer, const MediaSource::ReadOptions *o mBuffer->setObserver(this); mBuffer->add_ref(); - mBuffer->meta_data()->setInt32(kKeyIsSyncFrame, 1); (*buffer) = mBuffer; return OK; @@ -109,15 +160,17 @@ status_t OMXSource::stop() { quitFlag_ = true; if (Log::isVerbose()) Log_v("quit flag to true"); if (Log::isDebug()) Log_d("delete pbuffer: %d", pbuffers_.size()); - while (!pbuffers_.empty()) { - MediaBuffer* buffer = pbuffers_.front(); - pbuffers_.pop(); - buffer->release(); - } + clearQueuedBuffers(); cond_.notify_one(); } if (Log::isDebug()) Log_d("stopped"); return OK; } - +void OMXSource::clearQueuedBuffers() { + while (!pbuffers_.empty()) { + MediaBuffer* buffer = pbuffers_.front(); + pbuffers_.pop(); + buffer->release(); + } +} diff --git a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h index d586d8fe..bc35f040 100644 --- a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h +++ b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,9 @@ class OMXSource : public MediaSource, public MediaBufferObserver { std::condition_variable cond_; bool quitFlag_; + bool waitingForSync_; + std::size_t droppedBuffers_; MediaBuffer* nextBuffer(); + void clearQueuedBuffers(); }; From 2bd1f5c7c03385b8027b95f0d774172fce2dfa7c Mon Sep 17 00:00:00 2001 From: Praveen <16594062+14praveenk@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:15:09 +0100 Subject: [PATCH 2/2] Avoid decoder freeze while waiting for IDR --- .../cpp/libomxvideocodec-jni/OMXSource.cpp | 38 +++++++++++++++---- .../main/cpp/libomxvideocodec-jni/OMXSource.h | 3 ++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp index 1f6ac84c..ca4d8718 100644 --- a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp +++ b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp @@ -14,6 +14,8 @@ namespace { constexpr std::size_t kMaxQueuedBuffers = 8; constexpr int64_t kMaxQueuedDurationUs = 250000; +constexpr std::size_t kMaxSyncWaitBuffers = 8; +constexpr auto kMaxSyncWait = std::chrono::milliseconds(250); } // namespace @@ -21,7 +23,8 @@ constexpr int64_t kMaxQueuedDurationUs = 250000; // https://vec.io/posts/use-android-hardware-decoder-with-omxcodec-in-ndk // https://stackoverflow.com/questions/9832503/android-include-native-stagefright-features-in-my-own-project OMXSource::OMXSource(int width, int height, int fps): - format_(nullptr), quitFlag_(false), waitingForSync_(false), droppedBuffers_(0) + format_(nullptr), quitFlag_(false), waitingForSync_(false), droppedBuffers_(0), + syncWaitDroppedBuffers_(0) { int32_t bufferSize = (width * height * 3) / 2; @@ -68,27 +71,48 @@ void OMXSource::queueBuffer(MediaBuffer* buffer){ durationExceeded = timestampUs - oldestTimestampUs > kMaxQueuedDurationUs; } - if (pbuffers_.size() >= kMaxQueuedBuffers || durationExceeded) { + if (!waitingForSync_ && + (pbuffers_.size() >= kMaxQueuedBuffers || durationExceeded)) { droppedBuffers_ += pbuffers_.size(); clearQueuedBuffers(); waitingForSync_ = true; + syncWaitDroppedBuffers_ = 0; + syncWaitStarted_ = std::chrono::steady_clock::now(); if (Log::isWarn()) { - Log_w("video decoder backlog exceeded %lld us; dropping until the next IDR frame", + Log_w("video decoder backlog exceeded %lld us; briefly waiting for an IDR frame", static_cast(kMaxQueuedDurationUs)); } } if (waitingForSync_ && !nalInfo.hasCodecConfig && !nalInfo.hasIdr) { - ++droppedBuffers_; - buffer->release(); - return; + const bool waitExpired = syncWaitDroppedBuffers_ >= kMaxSyncWaitBuffers || + std::chrono::steady_clock::now() - syncWaitStarted_ >= kMaxSyncWait; + if (!waitExpired) { + ++droppedBuffers_; + ++syncWaitDroppedBuffers_; + buffer->release(); + return; + } + + // Some transitions do not emit an IDR promptly. The decoder still owns its + // previous reference frames, so resume feeding instead of freezing forever. + waitingForSync_ = false; + syncWaitDroppedBuffers_ = 0; + if (Log::isWarn()) { + Log_w("video decoder IDR wait expired after dropping %d buffers; resuming stream", + static_cast(droppedBuffers_)); + } + droppedBuffers_ = 0; + buffer->meta_data()->setInt32(kKeyIsSyncFrame, 1); + } else { + buffer->meta_data()->setInt32(kKeyIsSyncFrame, nalInfo.hasIdr ? 1 : 0); } - buffer->meta_data()->setInt32(kKeyIsSyncFrame, nalInfo.hasIdr ? 1 : 0); pbuffers_.push(buffer); if (nalInfo.hasIdr && waitingForSync_) { waitingForSync_ = false; + syncWaitDroppedBuffers_ = 0; if (Log::isWarn()) { Log_w("video decoder resynchronized after dropping %d buffers", static_cast(droppedBuffers_)); diff --git a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h index bc35f040..a2b1ec1e 100644 --- a/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h +++ b/omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,8 @@ class OMXSource : public MediaSource, public MediaBufferObserver { bool quitFlag_; bool waitingForSync_; std::size_t droppedBuffers_; + std::size_t syncWaitDroppedBuffers_; + std::chrono::steady_clock::time_point syncWaitStarted_; MediaBuffer* nextBuffer(); void clearQueuedBuffers();