Improve video decoder resync handling to prevent freezes (fix for Waze accumulating latency)

aasdk/src/main/cpp/libaasdk-jni/messenger/Timestamp.cpp

@@ -1,7 +1,6 @@
 #include <boost/endian/conversion.hpp>
 #include <messenger/Timestamp.hpp>
-#include <Log.h>
-#include <chrono>
+#include <cstring>
 
 namespace aasdk
 {
@@ -15,12 +14,15 @@ Timestamp::Timestamp(ValueType stamp)
 }
 
 Timestamp::Timestamp(const common::DataConstBuffer& buffer)
+    : stamp_(0)
 {
-//    const ValueType& timestampBig = reinterpret_cast<const ValueType&>(buffer.cdata[0]); // generates sigbus on release build
-//    auto timestampBig = reinterpret_cast<const ValueType>(&buffer.cdata[0]); // result in wrong timestamp
-//    stamp_ = boost::endian::big_to_native(timestampBig);
-    stamp_ = std::chrono::system_clock::now().time_since_epoch().count();
-    Log_v("stamp_ %lld", stamp_);
+    if (buffer.size < sizeof(ValueType)) {
+        return;
+    }
+
+    ValueType timestampBig;
+    std::memcpy(&timestampBig, buffer.cdata, sizeof(timestampBig));
+    stamp_ = boost::endian::big_to_native(timestampBig);
 }
 
 common::Data Timestamp::getData() const

app/src/main/java/it/smg/hu/projection/NativeVideoOutput.java

@@ -20,8 +20,8 @@ public class NativeVideoOutput extends VideoOutput implements Runnable {
     private static final String TAG = "NativeVideoOutput";
 
     private MediaCodec codec_;
-    private boolean configured_;
-    private boolean running_;
+    private volatile boolean configured_;
+    private volatile boolean running_;
     private Thread codecThread_;
 
     public NativeVideoOutput(SurfaceView surfaceView){
@@ -57,10 +57,10 @@ public boolean init() {
 //            format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
             format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 655360);
             codec_.configure(format, surface, null, 0);
-            configured_ = true;
             codec_.start();
-            codecThread_.start();
+            configured_ = true;
             running_ = true;
+            codecThread_.start();
             return true;
         } catch (IOException e) {
             return false;
@@ -113,17 +113,9 @@ public void run() {
         MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
         while (running_) {
             if (configured_) {
-                int index = codec_.dequeueOutputBuffer(info, 0);
+                int index = codec_.dequeueOutputBuffer(info, 10000);
                 if (index >= 0) {
                     if (Log.isVerbose()) Log.v(TAG, "outputBufferIndex: " + index);
-                    ByteBuffer buffer = null;
-                    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
-                        buffer = codec_.getOutputBuffer(index);
-                    } else {
-                        buffer = codec_.getOutputBuffers()[index];
-                    }
-                    if (Log.isVerbose()) Log.v(TAG, "outputBuffer: " + buffer);
-
                     // setting true is telling system to render frame onto Surface
                     codec_.releaseOutputBuffer(index, true);
                 }

omxvideocodec/src/main/cpp/libomxvideocodec-jni/H264NalParser.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace h264 {
+
+struct NalInfo {
+    bool hasCodecConfig = false;
+    bool hasIdr = false;
+};
+
+inline void addNalType(NalInfo& info, uint8_t header) {
+    const uint8_t type = header & 0x1f;
+    info.hasCodecConfig = info.hasCodecConfig || type == 7 || type == 8;
+    info.hasIdr = info.hasIdr || type == 5;
+}
+
+inline NalInfo inspectNalUnits(const uint8_t* data, std::size_t size) {
+    NalInfo info;
+    if (data == nullptr || size == 0) {
+        return info;
+    }
+
+    bool foundAnnexB = false;
+
+    for (std::size_t i = 0; i + 3 < size; ++i) {
+        std::size_t headerOffset = 0;
+        if (data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1) {
+            headerOffset = i + 3;
+        } else if (i + 4 < size && data[i] == 0 && data[i + 1] == 0 &&
+                   data[i + 2] == 0 && data[i + 3] == 1) {
+            headerOffset = i + 4;
+        }
+
+        if (headerOffset < size && headerOffset != 0) {
+            addNalType(info, data[headerOffset]);
+            foundAnnexB = true;
+            i = headerOffset;
+        }
+    }
+
+    if (foundAnnexB) {
+        return info;
+    }
+
+    // Android Auto normally sends Annex B, but accept four-byte AVCC lengths too.
+    std::size_t offset = 0;
+    bool foundAvcc = false;
+    while (offset + 4 < size) {
+        const uint32_t nalSize = (static_cast<uint32_t>(data[offset]) << 24) |
+                                 (static_cast<uint32_t>(data[offset + 1]) << 16) |
+                                 (static_cast<uint32_t>(data[offset + 2]) << 8) |
+                                 static_cast<uint32_t>(data[offset + 3]);
+        if (nalSize == 0 || nalSize > size - offset - 4) {
+            break;
+        }
+
+        addNalType(info, data[offset + 4]);
+        foundAvcc = true;
+        offset += 4 + nalSize;
+    }
+
+    if (!foundAvcc && size > 0) {
+        addNalType(info, data[0]);
+    }
+
+    return info;
+}
+
+} // namespace h264

omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXDecoder.cpp

@@ -71,7 +71,7 @@ OMXDecoder::~OMXDecoder(){
 status_t OMXDecoder::read(){
     if (Log::isVerbose()) Log_v("read");
 
-    MediaBuffer *videoBuffer;
+    MediaBuffer *videoBuffer = nullptr;
     MediaSource::ReadOptions options;
     options.setLateBy(0);
     status_t ret = decoder_->read(&videoBuffer, &options);
@@ -81,10 +81,8 @@ status_t OMXDecoder::read(){
         if (videoBuffer->range_length() > 0) {
             if (Log::isVerbose()) Log_v("videobuffer length %d", videoBuffer->range_length());
             sp<MetaData> metaData = videoBuffer->meta_data();
-            int64_t timeUs = 0;
-            metaData->findInt64(kKeyTime, &timeUs);
             native_window_set_buffers_timestamp(nativeWindow_.get(),
-                                                timeUs); //NATIVE_WINDOW_TIMESTAMP_AUTO
+                                                NATIVE_WINDOW_TIMESTAMP_AUTO);
             ret = nativeWindow_->queueBuffer(nativeWindow_.get(),
                                              videoBuffer->graphicBuffer().get());
             if (ret == 0) {
@@ -103,5 +101,3 @@ status_t OMXDecoder::read(){
     return ret;
 }
 
-
-

omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.cpp

@@ -1,4 +1,5 @@
 #include "OMXSource.h"
+#include "H264NalParser.h"
 #include "Log.h"
 
 #include <media/stagefright/MediaDefs.h>
@@ -9,11 +10,21 @@
 
 using namespace android;
 
+namespace {
+
+constexpr std::size_t kMaxQueuedBuffers = 8;
+constexpr int64_t kMaxQueuedDurationUs = 250000;
+constexpr std::size_t kMaxSyncWaitBuffers = 8;
+constexpr auto kMaxSyncWait = std::chrono::milliseconds(250);
+
+} // namespace
+
 // https://www.programmersought.com/article/87712558400/
 // https://vec.io/posts/use-android-hardware-decoder-with-omxcodec-in-ndk
 // https://stackoverflow.com/questions/9832503/android-include-native-stagefright-features-in-my-own-project
 OMXSource::OMXSource(int width, int height, int fps):
-        format_(nullptr), quitFlag_(false)
+        format_(nullptr), quitFlag_(false), waitingForSync_(false), droppedBuffers_(0),
+        syncWaitDroppedBuffers_(0)
 {
 
     int32_t bufferSize = (width * height * 3) / 2;
@@ -41,11 +52,76 @@ void OMXSource::queueBuffer(MediaBuffer* buffer){
     if (Log::isVerbose()) Log_v("add buffer to queue");
     std::unique_lock<std::mutex> l(mutex_);
 
-    if (!quitFlag_) {
-        pbuffers_.push(buffer);
-        if (Log::isVerbose()) Log_v("queueBuffer new size %d", pbuffers_.size());
-        cond_.notify_one();
+    if (quitFlag_) {
+        buffer->release();
+        return;
+    }
+
+    const auto* bufferData = static_cast<const uint8_t*>(buffer->data());
+    const auto* data = bufferData == nullptr ? nullptr : bufferData + buffer->range_offset();
+    const h264::NalInfo nalInfo = h264::inspectNalUnits(data, buffer->range_length());
+    int64_t timestampUs = 0;
+    int64_t oldestTimestampUs = 0;
+    buffer->meta_data()->findInt64(kKeyTime, &timestampUs);
+
+    bool durationExceeded = false;
+    if (!pbuffers_.empty() && timestampUs > 0 &&
+        pbuffers_.front()->meta_data()->findInt64(kKeyTime, &oldestTimestampUs) &&
+        oldestTimestampUs > 0 && timestampUs > oldestTimestampUs) {
+        durationExceeded = timestampUs - oldestTimestampUs > kMaxQueuedDurationUs;
     }
+
+    if (!waitingForSync_ &&
+        (pbuffers_.size() >= kMaxQueuedBuffers || durationExceeded)) {
+        droppedBuffers_ += pbuffers_.size();
+        clearQueuedBuffers();
+        waitingForSync_ = true;
+        syncWaitDroppedBuffers_ = 0;
+        syncWaitStarted_ = std::chrono::steady_clock::now();
+        if (Log::isWarn()) {
+            Log_w("video decoder backlog exceeded %lld us; briefly waiting for an IDR frame",
+                  static_cast<long long>(kMaxQueuedDurationUs));
+        }
+    }
+
+    if (waitingForSync_ && !nalInfo.hasCodecConfig && !nalInfo.hasIdr) {
+        const bool waitExpired = syncWaitDroppedBuffers_ >= kMaxSyncWaitBuffers ||
+            std::chrono::steady_clock::now() - syncWaitStarted_ >= kMaxSyncWait;
+        if (!waitExpired) {
+            ++droppedBuffers_;
+            ++syncWaitDroppedBuffers_;
+            buffer->release();
+            return;
+        }
+
+        // Some transitions do not emit an IDR promptly. The decoder still owns its
+        // previous reference frames, so resume feeding instead of freezing forever.
+        waitingForSync_ = false;
+        syncWaitDroppedBuffers_ = 0;
+        if (Log::isWarn()) {
+            Log_w("video decoder IDR wait expired after dropping %d buffers; resuming stream",
+                  static_cast<int>(droppedBuffers_));
+        }
+        droppedBuffers_ = 0;
+        buffer->meta_data()->setInt32(kKeyIsSyncFrame, 1);
+    } else {
+        buffer->meta_data()->setInt32(kKeyIsSyncFrame, nalInfo.hasIdr ? 1 : 0);
+    }
+
+    pbuffers_.push(buffer);
+
+    if (nalInfo.hasIdr && waitingForSync_) {
+        waitingForSync_ = false;
+        syncWaitDroppedBuffers_ = 0;
+        if (Log::isWarn()) {
+            Log_w("video decoder resynchronized after dropping %d buffers",
+                  static_cast<int>(droppedBuffers_));
+        }
+        droppedBuffers_ = 0;
+    }
+
+    if (Log::isVerbose()) Log_v("queueBuffer new size %d", pbuffers_.size());
+    cond_.notify_one();
 }
 
 sp<MetaData>  OMXSource::getFormat(){
@@ -82,7 +158,6 @@ status_t OMXSource::read(MediaBuffer **buffer, const MediaSource::ReadOptions *o
 
     mBuffer->setObserver(this);
     mBuffer->add_ref();
-    mBuffer->meta_data()->setInt32(kKeyIsSyncFrame, 1);
     (*buffer) = mBuffer;
 
     return OK;
@@ -109,15 +184,17 @@ status_t OMXSource::stop() {
         quitFlag_ = true;
         if (Log::isVerbose()) Log_v("quit flag to true");
         if (Log::isDebug()) Log_d("delete pbuffer: %d", pbuffers_.size());
-        while (!pbuffers_.empty()) {
-            MediaBuffer* buffer = pbuffers_.front();
-            pbuffers_.pop();
-            buffer->release();
-        }
+        clearQueuedBuffers();
         cond_.notify_one();
     }
     if (Log::isDebug()) Log_d("stopped");
     return OK;
 }
 
-
+void OMXSource::clearQueuedBuffers() {
+    while (!pbuffers_.empty()) {
+        MediaBuffer* buffer = pbuffers_.front();
+        pbuffers_.pop();
+        buffer->release();
+    }
+}

omxvideocodec/src/main/cpp/libomxvideocodec-jni/OMXSource.h

@@ -4,6 +4,8 @@
 #include <media/stagefright/MetaData.h>
 #include <utils/RefBase.h>
 #include <media/stagefright/MediaBufferGroup.h>
+#include <chrono>
+#include <cstddef>
 #include <queue>
 #include <thread>
 #include <Data.hpp>
@@ -44,6 +46,11 @@ class OMXSource : public MediaSource, public MediaBufferObserver {
     std::condition_variable cond_;
 
     bool quitFlag_;
+    bool waitingForSync_;
+    std::size_t droppedBuffers_;
+    std::size_t syncWaitDroppedBuffers_;
+    std::chrono::steady_clock::time_point syncWaitStarted_;
 
     MediaBuffer* nextBuffer();
+    void clearQueuedBuffers();
 };