Switch to async mediacodec APIs

This also fixes an overflow bug when calculating the presentation
timestamp by making input_fragment_index_ a 64-bit int.

Test: manually check that playback works
Bug: 146030443

Change-Id: Ied31c00fcab11d1af35195c4848fe9557b5bee3a
(cherry picked from commit beaff8e102e0f57e35cb11add86f5270f045fbae)

tests/c2_e2e_test/jni/mediacodec_decoder.cpp

@@ -114,9 +114,41 @@ std::unique_ptr<MediaCodecDecoder> MediaCodecDecoder::Create(const std::string&
         return nullptr;
     }
 
-    return std::unique_ptr<MediaCodecDecoder>(
+    auto ret = std::unique_ptr<MediaCodecDecoder>(
             new MediaCodecDecoder(codec, std::move(encoded_data_helper), type, video_size,
                                   frame_rate, surface, render_on_release, loop));
+
+    AMediaCodecOnAsyncNotifyCallback cb{
+            .onAsyncInputAvailable =
+                    [](AMediaCodec* codec, void* userdata, int32_t index) {
+                        reinterpret_cast<MediaCodecDecoder*>(userdata)->OnAsyncInputAvailable(
+                                index);
+                    },
+            .onAsyncOutputAvailable =
+                    [](AMediaCodec* codec, void* userdata, int32_t index,
+                       AMediaCodecBufferInfo* buffer_info) {
+                        reinterpret_cast<MediaCodecDecoder*>(userdata)->OnAsyncOutputAvailable(
+                                index, buffer_info);
+                    },
+            .onAsyncFormatChanged =
+                    [](AMediaCodec* codec, void* userdata, AMediaFormat* format) {
+                        reinterpret_cast<MediaCodecDecoder*>(userdata)->OnAsyncFormatChanged(
+                                format);
+                    },
+            .onAsyncError =
+                    [](AMediaCodec* codec, void* userdata, media_status_t error, int32_t code,
+                       const char* detail) {
+                        ALOGE("Error %d (%d) %s", error, code, detail);
+                        assert(false);
+                    }};
+
+    auto status = AMediaCodec_setAsyncNotifyCallback(codec, cb, ret.get());
+    if (status != AMEDIA_OK) {
+        ALOGE("Failed to set async callback.");
+        return nullptr;
+    }
+
+    return ret;
 }
 
 MediaCodecDecoder::MediaCodecDecoder(AMediaCodec* codec,
@@ -146,6 +178,24 @@ void MediaCodecDecoder::AddOutputFormatChangedCb(const OutputFormatChangedCb& cb
     output_format_changed_cbs_.push_back(cb);
 }
 
+void MediaCodecDecoder::OnAsyncInputAvailable(int32_t idx) {
+    std::lock_guard<std::mutex> lock(event_queue_mut_);
+    event_queue_.push({.type = INPUT_AVAILABLE, .idx = idx});
+    event_queue_cv_.notify_one();
+}
+
+void MediaCodecDecoder::OnAsyncOutputAvailable(int32_t idx, AMediaCodecBufferInfo* info) {
+    std::lock_guard<std::mutex> lock(event_queue_mut_);
+    event_queue_.push({.type = OUTPUT_AVAILABLE, .idx = idx, .info = *info});
+    event_queue_cv_.notify_one();
+}
+
+void MediaCodecDecoder::OnAsyncFormatChanged(AMediaFormat* format) {
+    std::lock_guard<std::mutex> lock(event_queue_mut_);
+    event_queue_.push({.type = FORMAT_CHANGED});
+    event_queue_cv_.notify_one();
+}
+
 void MediaCodecDecoder::Rewind() {
     encoded_data_helper_->Rewind();
     input_fragment_index_ = 0;
@@ -179,85 +229,56 @@ bool MediaCodecDecoder::Start() {
 
 bool MediaCodecDecoder::Decode() {
     while (!output_done_) {
-        size_t retries = 0;
-        bool success = false;
-
-        // It will keep retrying until one output buffer is dequeued successfully.
-        // On each retry we would like to enqueue input buffers as fast as possible.
-        // The retry loop will break as failure if maxmimum retries are reached or
-        // errors returned from enqueue input buffer or dequeue output buffer.
-        while (retries < kTimeoutMaxRetries && !success) {
-            if (!EnqueueInputBuffers()) return false;
-
-            switch (DequeueOutputBuffer()) {
-            case DequeueStatus::RETRY:
-                retries++;
-                break;
-            case DequeueStatus::SUCCESS:
-                success = true;
-                break;
-            case DequeueStatus::FAILURE:
-                return false;
+        CodecEvent evt;
+        {
+            std::unique_lock<std::mutex> lock(event_queue_mut_);
+            while (event_queue_.empty()) {
+                event_queue_cv_.wait(lock);
             }
+            evt = event_queue_.front();
+            event_queue_.pop();
         }
 
-        if (retries >= kTimeoutMaxRetries) {
-            ALOGE("Decoder did not produce an output buffer after %zu retries", kTimeoutMaxRetries);
+        bool success;
+        switch (evt.type) {
+        case INPUT_AVAILABLE:
+            success = EnqueueInputBuffers(evt.idx);
+            break;
+        case OUTPUT_AVAILABLE:
+            success = DequeueOutputBuffer(evt.idx, evt.info);
+            break;
+        case FORMAT_CHANGED:
+            success = GetOutputFormat();
+            break;
         }
-        if (!success) return false;
+        assert(success);
     }
     return true;
 }
 
-bool MediaCodecDecoder::EnqueueInputBuffers() {
-    ssize_t index;
-    while (!input_done_) {
-        index = AMediaCodec_dequeueInputBuffer(codec_, kTimeoutWaitForInputUs);
-        if (index == AMEDIACODEC_INFO_TRY_AGAIN_LATER)
-            return true;  // no available input buffers, try next time
-
-        if (index < 0) {
-            ALOGE("Unknown error while dequeueInputBuffer: %zd", index);
-            return false;
-        }
+bool MediaCodecDecoder::EnqueueInputBuffers(int32_t index) {
+    if (index < 0) {
+        ALOGE("Unknown error while dequeueInputBuffer: %zd", index);
+        return false;
+    }
 
-        if (looping_ && encoded_data_helper_->ReachEndOfStream()) {
-            encoded_data_helper_->Rewind();
-        }
+    if (looping_ && encoded_data_helper_->ReachEndOfStream()) {
+        encoded_data_helper_->Rewind();
+    }
 
-        if (encoded_data_helper_->ReachEndOfStream()) {
-            if (!FeedEOSInputBuffer(index)) return false;
-            input_done_ = true;
-        } else {
-            if (!FeedInputBuffer(index)) return false;
-        }
+    if (encoded_data_helper_->ReachEndOfStream()) {
+        if (!FeedEOSInputBuffer(index)) return false;
+        input_done_ = true;
+    } else {
+        if (!FeedInputBuffer(index)) return false;
     }
     return true;
 }
 
-MediaCodecDecoder::DequeueStatus MediaCodecDecoder::DequeueOutputBuffer() {
-    AMediaCodecBufferInfo info;
-    ssize_t index = AMediaCodec_dequeueOutputBuffer(codec_, &info, kTimeoutWaitForOutputUs);
-
-    switch (index) {
-    case AMEDIACODEC_INFO_TRY_AGAIN_LATER:
-        ALOGV("Try again later is reported");
-        return DequeueStatus::RETRY;
-    case AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED:
-        ALOGV("Output buffers changed");
-        return DequeueStatus::RETRY;
-    case AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED:
-        ALOGV("Output format changed");
-        if (GetOutputFormat())
-            return DequeueStatus::SUCCESS;
-        else
-            return DequeueStatus::FAILURE;
-    default:
-        if (index < 0) {
-            ALOGE("Unknown error while dequeueOutputBuffer: %zd", index);
-            return DequeueStatus::FAILURE;
-        }
-        break;
+bool MediaCodecDecoder::DequeueOutputBuffer(int32_t index, AMediaCodecBufferInfo info) {
+    if (index < 0) {
+        ALOGE("Unknown error while dequeueOutputBuffer: %zd", index);
+        return false;
     }
 
     if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) output_done_ = true;
@@ -272,9 +293,9 @@ MediaCodecDecoder::DequeueStatus MediaCodecDecoder::DequeueOutputBuffer() {
         }
     }
 
-    if (!ReceiveOutputBuffer(index, info, render_on_release_)) return DequeueStatus::FAILURE;
+    if (!ReceiveOutputBuffer(index, info, render_on_release_)) return false;
 
-    return DequeueStatus::SUCCESS;
+    return true;
 }
 
 bool MediaCodecDecoder::Stop() {

tests/c2_e2e_test/jni/mediacodec_decoder.h

@@ -5,7 +5,9 @@
 #ifndef C2_E2E_TEST_MEDIACODEC_DECODER_H_
 #define C2_E2E_TEST_MEDIACODEC_DECODER_H_
 
+#include <condition_variable>
 #include <memory>
+#include <mutex>
 #include <queue>
 #include <string>
 #include <vector>
@@ -64,7 +66,18 @@ public:
 
     int32_t dropped_frame_count() const { return drop_frame_count_; }
 
+    void OnAsyncInputAvailable(int32_t idx);
+    void OnAsyncOutputAvailable(int32_t idx, AMediaCodecBufferInfo* info);
+    void OnAsyncFormatChanged(AMediaFormat* format);
+
 private:
+    enum CodecEventType { INPUT_AVAILABLE, OUTPUT_AVAILABLE, FORMAT_CHANGED };
+    struct CodecEvent {
+        CodecEventType type;
+        int32_t idx;
+        AMediaCodecBufferInfo info;
+    };
+
     MediaCodecDecoder(AMediaCodec* codec, std::unique_ptr<EncodedDataHelper> encoded_data_helper,
                       VideoCodecType type, const Size& size, int frame_rate, ANativeWindow* surface,
                       bool renderOnRelease, bool loop);
@@ -73,10 +86,10 @@ private:
     enum class DequeueStatus { RETRY, SUCCESS, FAILURE };
 
     // Fill all available input buffers and enqueue.
-    bool EnqueueInputBuffers();
+    bool EnqueueInputBuffers(int32_t idx);
 
     // Try to dequeue one output buffer and return DequeueStatus.
-    DequeueStatus DequeueOutputBuffer();
+    bool DequeueOutputBuffer(int32_t idx, AMediaCodecBufferInfo info);
 
     // Read the sample data from AMediaExtractor or CSD data and feed into the
     // input buffer.
@@ -120,7 +133,7 @@ private:
 
     // The fragment index that indicates which frame is sent to the decoder at
     // next round.
-    int input_fragment_index_ = 0;
+    int64_t input_fragment_index_ = 0;
     // The total number of received output buffers. Only used for logging.
     int received_outputs_ = 0;
 
@@ -136,6 +149,10 @@ private:
     int32_t drop_frame_count_ = 0;
 
     std::atomic<bool> looping_;
+
+    std::queue<CodecEvent> event_queue_;  // guarded by event_queue_mut_
+    std::mutex event_queue_mut_;
+    std::condition_variable event_queue_cv_;
 };
 
 }  // namespace android