[FFmpeg-devel,1/2] avcodec/mediacodec_wrapper: add async mode support

diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c
index f2fb392329..2f4a3855e4 100644
--- a/libavcodec/mediacodec_wrapper.c
+++ b/libavcodec/mediacodec_wrapper.c
@@ -1848,6 +1848,14 @@  static int mediacodec_jni_signalEndOfInputStream(FFAMediaCodec *ctx)
     return 0;
 }
 
+static int mediacodec_jni_setAsyncNotifyCallback(FFAMediaCodec *codec,
+                                                 const FFAMediaCodecOnAsyncNotifyCallback *callback,
+                                                 void *userdata)
+{
+    av_log(codec, AV_LOG_ERROR, "Doesn't support aync mode with JNI, please try ndk_codec=1\n");
+    return AVERROR(ENOSYS);
+}
+
 static const FFAMediaFormat media_format_jni = {
     .class = &amediaformat_class,
 
@@ -1907,6 +1915,8 @@  static const FFAMediaCodec media_codec_jni = {
     .getConfigureFlagEncode = mediacodec_jni_getConfigureFlagEncode,
     .cleanOutputBuffers = mediacodec_jni_cleanOutputBuffers,
     .signalEndOfInputStream = mediacodec_jni_signalEndOfInputStream,
+
+    .setAsyncNotifyCallback = mediacodec_jni_setAsyncNotifyCallback,
 };
 
 typedef struct FFAMediaFormatNdk {
@@ -1945,6 +1955,9 @@  typedef struct FFAMediaCodecNdk {
     AMediaCodec *impl;
     ANativeWindow *window;
 
+    FFAMediaCodecOnAsyncNotifyCallback async_cb;
+    void *async_userdata;
+
     AMediaCodec* (*createCodecByName)(const char *name);
     AMediaCodec* (*createDecoderByType)(const char *mime_type);
     AMediaCodec* (*createEncoderByType)(const char *mime_type);
@@ -1980,6 +1993,11 @@  typedef struct FFAMediaCodecNdk {
     // Available since API level 26.
     media_status_t (*setInputSurface)(AMediaCodec*, ANativeWindow *);
     media_status_t (*signalEndOfInputStream)(AMediaCodec *);
+
+    // Available since API level 28
+    media_status_t (*setAsyncNotifyCallback)(AMediaCodec *,
+                                             struct AMediaCodecOnAsyncNotifyCallback callback,
+                                             void *userdata);
 } FFAMediaCodecNdk;
 
 static const FFAMediaFormat media_format_ndk;
@@ -1995,6 +2013,32 @@  static const AVClass amediacodec_ndk_class = {
     .version    = LIBAVUTIL_VERSION_INT,
 };
 
+static int media_status_to_error(media_status_t status)
+{
+    switch (status) {
+    case AMEDIA_OK:
+        return 0;
+    case AMEDIACODEC_ERROR_INSUFFICIENT_RESOURCE:
+        return AVERROR(ENOMEM);
+    case AMEDIA_ERROR_MALFORMED:
+        return AVERROR_INVALIDDATA;
+    case AMEDIA_ERROR_UNSUPPORTED:
+        return AVERROR(ENOTSUP);
+    case AMEDIA_ERROR_INVALID_PARAMETER:
+        return AVERROR(EINVAL);
+    case AMEDIA_ERROR_INVALID_OPERATION:
+        return AVERROR(EOPNOTSUPP);
+    case AMEDIA_ERROR_END_OF_STREAM:
+        return AVERROR_EOF;
+    case AMEDIA_ERROR_IO:
+        return AVERROR(EIO);
+    case AMEDIA_ERROR_WOULD_BLOCK:
+        return AVERROR(EWOULDBLOCK);
+    default:
+        return AVERROR_EXTERNAL;
+    }
+}
+
 static FFAMediaFormat *mediaformat_ndk_create(AMediaFormat *impl)
 {
     FFAMediaFormatNdk *format = av_mallocz(sizeof(*format));
@@ -2239,6 +2283,8 @@  static inline FFAMediaCodec *ndk_codec_create(int method, const char *arg) {
     GET_SYMBOL(setInputSurface, 0)
     GET_SYMBOL(signalEndOfInputStream, 0)
 
+    GET_SYMBOL(setAsyncNotifyCallback, 0)
+
 #undef GET_SYMBOL
 
     switch (method) {
@@ -2513,6 +2559,85 @@  static int mediacodec_ndk_signalEndOfInputStream(FFAMediaCodec *ctx)
     return 0;
 }
 
+static void mediacodec_ndk_onInputAvailable(AMediaCodec *impl, void *userdata, int32_t index)
+{
+    FFAMediaCodecNdk *codec = userdata;
+    codec->async_cb.onAsyncInputAvailable((FFAMediaCodec *) codec, codec->async_userdata, index);
+}
+
+static void mediacodec_ndk_onOutputAvailable(AMediaCodec *impl,
+                                             void *userdata,
+                                             int32_t index,
+                                             AMediaCodecBufferInfo *buffer_info)
+{
+    FFAMediaCodecNdk *codec = userdata;
+    FFAMediaCodecBufferInfo info = {
+            .offset = buffer_info->offset,
+            .size = buffer_info->size,
+            .presentationTimeUs = buffer_info->presentationTimeUs,
+            .flags = buffer_info->flags,
+    };
+
+    codec->async_cb.onAsyncOutputAvailable(&codec->api, codec->async_userdata, index, &info);
+}
+
+static void mediacodec_ndk_onFormatChanged(AMediaCodec *impl, void *userdata, AMediaFormat *format)
+{
+    FFAMediaCodecNdk *codec = userdata;
+    FFAMediaFormat *media_format = mediaformat_ndk_create(format);
+    if (!media_format)
+        return;
+
+    codec->async_cb.onAsyncFormatChanged(&codec->api, codec->async_userdata, media_format);
+    ff_AMediaFormat_delete(media_format);
+}
+
+static void mediacodec_ndk_onError(AMediaCodec *impl, void *userdata,
+                                   media_status_t status,
+                                   int32_t actionCode,
+                                   const char *detail)
+{
+    FFAMediaCodecNdk *codec = userdata;
+    int error = media_status_to_error(status);
+
+    codec->async_cb.onAsyncError(&codec->api, codec->async_userdata, error, detail);
+}
+
+static int mediacodec_ndk_setAsyncNotifyCallback(FFAMediaCodec *ctx,
+                                                 const FFAMediaCodecOnAsyncNotifyCallback *callback,
+                                                 void *userdata)
+{
+    FFAMediaCodecNdk *codec = (FFAMediaCodecNdk *)ctx;
+    struct AMediaCodecOnAsyncNotifyCallback cb = {
+            .onAsyncInputAvailable = mediacodec_ndk_onInputAvailable,
+            .onAsyncOutputAvailable = mediacodec_ndk_onOutputAvailable,
+            .onAsyncFormatChanged = mediacodec_ndk_onFormatChanged,
+            .onAsyncError = mediacodec_ndk_onError,
+    };
+    media_status_t status;
+
+    if (!codec->setAsyncNotifyCallback) {
+        av_log(codec, AV_LOG_ERROR, "setAsyncNotifyCallback unavailable\n");
+        return AVERROR(ENOSYS);
+    }
+
+    if (!callback ||
+        !callback->onAsyncInputAvailable || !callback->onAsyncOutputAvailable ||
+        !callback->onAsyncFormatChanged || !callback->onAsyncError)
+        return AVERROR(EINVAL);
+
+    codec->async_cb = *callback;
+    codec->async_userdata = userdata;
+
+    status = codec->setAsyncNotifyCallback(codec->impl, cb, codec);
+    if (status != AMEDIA_OK) {
+        av_log(codec, AV_LOG_ERROR, "setAsyncNotifyCallback failed, %d\n", status);
+        return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
 static const FFAMediaFormat media_format_ndk = {
     .class = &amediaformat_ndk_class,
 
@@ -2574,6 +2699,8 @@  static const FFAMediaCodec media_codec_ndk = {
     .getConfigureFlagEncode = mediacodec_ndk_getConfigureFlagEncode,
     .cleanOutputBuffers = mediacodec_ndk_cleanOutputBuffers,
     .signalEndOfInputStream = mediacodec_ndk_signalEndOfInputStream,
+
+    .setAsyncNotifyCallback = mediacodec_ndk_setAsyncNotifyCallback,
 };
 
 FFAMediaFormat *ff_AMediaFormat_new(int ndk)
diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h
index 11a4260497..084e9b3638 100644
--- a/libavcodec/mediacodec_wrapper.h
+++ b/libavcodec/mediacodec_wrapper.h
@@ -178,6 +178,19 @@  struct FFAMediaCodecBufferInfo {
 typedef struct FFAMediaCodecBufferInfo FFAMediaCodecBufferInfo;
 
 typedef struct FFAMediaCodec FFAMediaCodec;
+
+typedef struct FFAMediaCodecOnAsyncNotifyCallback {
+    void (*onAsyncInputAvailable)(FFAMediaCodec *codec, void *userdata, int32_t index);
+
+    void (*onAsyncOutputAvailable)(FFAMediaCodec *codec, void *userdata,
+                                   int32_t index,
+                                   FFAMediaCodecBufferInfo *buffer_info);
+
+    void (*onAsyncFormatChanged)(FFAMediaCodec *codec, void *userdata, FFAMediaFormat *format);
+
+    void (*onAsyncError)(FFAMediaCodec *codec, void *userdata, int error, const char *detail);
+} FFAMediaCodecOnAsyncNotifyCallback;
+
 struct FFAMediaCodec {
     const AVClass *class;
 
@@ -219,6 +232,11 @@  struct FFAMediaCodec {
 
     // For encoder with FFANativeWindow as input.
     int (*signalEndOfInputStream)(FFAMediaCodec *);
+
+    // Introduced in Android API 28
+    int (*setAsyncNotifyCallback)(FFAMediaCodec *codec,
+                                  const FFAMediaCodecOnAsyncNotifyCallback *callback,
+                                  void *userdata);
 };
 
 static inline char *ff_AMediaCodec_getName(FFAMediaCodec *codec)
@@ -343,6 +361,13 @@  static inline int ff_AMediaCodec_signalEndOfInputStream(FFAMediaCodec *codec)
     return codec->signalEndOfInputStream(codec);
 }
 
+static inline int ff_AMediaCodec_setAsyncNotifyCallback(FFAMediaCodec *codec,
+                        const FFAMediaCodecOnAsyncNotifyCallback *callback,
+                        void *userdata)
+{
+    return codec->setAsyncNotifyCallback(codec, callback, userdata);
+}
+
 int ff_Build_SDK_INT(AVCodecContext *avctx);
 
 enum FFAMediaFormatColorRange {