From 44c347f5a2b6160abd8fc09a0255ee21a7ba6ee4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomas=20H=C3=A4rdin?= <git@haerdin.se>
Date: Wed, 4 Jan 2023 16:27:26 +0100
Subject: [PATCH 2/2] lavc/mediacodecenc: Probe actually supported color
formats using JNI
Fall back on trying to open the encoder if JNI doesn't work.
This patch has been released by Epic Games' legal department.
---
libavcodec/mediacodec_wrapper.c | 93 +++++++++++++++++++++++++++++++++
libavcodec/mediacodec_wrapper.h | 15 ++++++
libavcodec/mediacodecenc.c | 27 +++++++++-
3 files changed, 134 insertions(+), 1 deletion(-)
@@ -2,6 +2,7 @@
* Android MediaCodec Wrapper
*
* Copyright (c) 2015-2016 Matthieu Bouron <matthieu.bouron stupeflix.com>
+ * Modifications by Epic Games, Inc., 2023.
*
* This file is part of FFmpeg.
*
@@ -176,6 +177,7 @@ struct JNIAMediaCodecFields {
jmethodID release_id;
jmethodID get_output_format_id;
+ jmethodID get_codec_info_id;
jmethodID dequeue_input_buffer_id;
jmethodID queue_input_buffer_id;
@@ -228,6 +230,7 @@ static const struct FFJniField jni_amediacodec_mapping[] = {
{ "android/media/MediaCodec", "release", "()V", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, release_id), 1 },
{ "android/media/MediaCodec", "getOutputFormat", "()Landroid/media/MediaFormat;", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, get_output_format_id), 1 },
+ { "android/media/MediaCodec", "getCodecInfo", "()Landroid/media/MediaCodecInfo;", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, get_codec_info_id), 1 },
{ "android/media/MediaCodec", "dequeueInputBuffer", "(J)I", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, dequeue_input_buffer_id), 1 },
{ "android/media/MediaCodec", "queueInputBuffer", "(IIIJI)V", FF_JNI_METHOD, offsetof(struct JNIAMediaCodecFields, queue_input_buffer_id), 1 },
@@ -604,6 +607,96 @@ done:
return name;
}
+int ff_AMediaCodec_color_formats_intersect(FFAMediaCodec *codec, const char *mime,
+ const int *in_formats, int nin_formats,
+ int *out_formats, void *log_ctx)
+{
+ FFAMediaCodecJni *jni_codec = (FFAMediaCodecJni*)codec;
+ jobject info = NULL;
+ jobject capabilities = NULL;
+ JNIEnv *env = NULL;
+ jintArray color_formats = NULL;
+ int color_count, ret, *elems;
+ jboolean is_copy;
+ jstring jmime = NULL;
+ struct JNIAMediaCodecFields jfields = { 0 };
+ struct JNIAMediaCodecListFields jfields2 = { 0 };
+
+ // make sure we have a JVM
+ JNI_GET_ENV_OR_RETURN(env, log_ctx, -1);
+
+ // grab mappings
+ if ((ret = ff_jni_init_jfields(env, &jfields, jni_amediacodec_mapping, 0, log_ctx)) < 0) {
+ goto done;
+ }
+
+ if ((ret = ff_jni_init_jfields(env, &jfields2, jni_amediacodeclist_mapping, 0, log_ctx)) < 0) {
+ goto done;
+ }
+
+ // codec.getCodecInfo().getCapabilitiesForType(mime).colorFormats
+ ret = -1;
+ info = (*env)->CallObjectMethod(env, jni_codec->object, jfields.get_codec_info_id);
+ if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+ goto done;
+ }
+
+ // convert const char* to java.lang.String
+ jmime = ff_jni_utf_chars_to_jstring(env, mime, log_ctx);
+ if (!jmime) {
+ goto done;
+ }
+
+ capabilities = (*env)->CallObjectMethod(env, info, jfields2.get_codec_capabilities_id, jmime);
+ if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+ goto done;
+ }
+
+ color_formats = (*env)->GetObjectField(env, capabilities, jfields2.color_formats_id);
+ if (ff_jni_exception_check(env, 1, log_ctx) < 0) {
+ goto done;
+ }
+
+ color_count = (*env)->GetArrayLength(env, color_formats);
+ elems = (*env)->GetIntArrayElements(env, color_formats, &is_copy);
+ ret = 0;
+
+ // out_formats = intersect(in_formats, elems)
+ for (int i = 0; i < nin_formats; i++) {
+ for (int j = 0; j < color_count; j++) {
+ if (elems[j] == in_formats[i]) {
+ out_formats[ret++] = in_formats[i];
+ break;
+ }
+ }
+ }
+
+ (*env)->ReleaseIntArrayElements(env, color_formats, elems, JNI_ABORT);
+
+done:
+ // clean up
+ if (jmime) {
+ (*env)->DeleteLocalRef(env, jmime);
+ }
+
+ if (color_formats) {
+ (*env)->DeleteLocalRef(env, color_formats);
+ }
+
+ if (capabilities) {
+ (*env)->DeleteLocalRef(env, capabilities);
+ }
+
+ if (info) {
+ (*env)->DeleteLocalRef(env, info);
+ }
+
+ ff_jni_reset_jfields(env, &jfields, jni_amediacodec_mapping, 0, log_ctx);
+ ff_jni_reset_jfields(env, &jfields2, jni_amediacodeclist_mapping, 0, log_ctx);
+
+ return ret;
+}
+
static FFAMediaFormat *mediaformat_jni_new(void)
{
JNIEnv *env = NULL;
@@ -2,6 +2,7 @@
* Android MediaCodec Wrapper
*
* Copyright (c) 2015-2016 Matthieu Bouron <matthieu.bouron stupeflix.com>
+ * Modifications by Epic Games, Inc., 2023.
*
* This file is part of FFmpeg.
*
@@ -230,6 +231,20 @@ FFAMediaCodec* ff_AMediaCodec_createCodecByName(const char *name, int ndk);
FFAMediaCodec* ff_AMediaCodec_createDecoderByType(const char *mime_type, int ndk);
FFAMediaCodec* ff_AMediaCodec_createEncoderByType(const char *mime_type, int ndk);
+/**
+ * Intersects the given list of color formats with the formats actually supported by the device.
+ * @param[in] codec The codec to query
+ * @param[in] mime MIME format. Typically "video/avc" or "video/hevc"
+ * @param[in] in_formats Array of input color formats
+ * @param[in] nin_formats Number of elements in in_formats
+ * @param[out] out_formats Computed intersection of in_formats and supported formats
+ * @param[in] log_ctx Logging context
+ * @return Size of out_formats or negative in case of error
+ */
+int ff_AMediaCodec_color_formats_intersect(FFAMediaCodec *codec, const char *mime,
+ const int *in_formats, int nin_formats,
+ int *out_formats, void *log_ctx);
+
static inline int ff_AMediaCodec_configure(FFAMediaCodec *codec,
const FFAMediaFormat *format,
FFANativeWindow *surface,
@@ -90,6 +90,12 @@ static const struct {
{ COLOR_FormatSurface, AV_PIX_FMT_MEDIACODEC },
};
+static const int in_formats[] = {
+ COLOR_FormatYUV420Planar,
+ COLOR_FormatYUV420SemiPlanar,
+ COLOR_FormatSurface,
+};
+
// filled in by mediacodec_init_static_data()
static enum AVPixelFormat probed_pix_fmts[FF_ARRAY_ELEMS(color_formats)+1];
@@ -535,7 +541,7 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec)
{
const char *codec_mime = ffcodec->p.id == AV_CODEC_ID_H264 ? "video/avc" : "video/hevc";
FFAMediaCodec *codec;
- int num_pix_fmts = 0;
+ int num_pix_fmts, out_formats[FF_ARRAY_ELEMS(in_formats)];
int use_ndk_codec = !av_jni_get_java_vm(NULL);
if (!(codec = ff_AMediaCodec_createEncoderByType(codec_mime, use_ndk_codec))) {
@@ -543,6 +549,24 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec)
return;
}
+ // attempt to query color formats using JNI
+ // fall back on NDK-compatible method below if this fails
+ if ((num_pix_fmts = ff_AMediaCodec_color_formats_intersect(
+ codec, codec_mime, in_formats, FF_ARRAY_ELEMS(in_formats),
+ out_formats, NULL)) >= 0) {
+ // translate color formats to pixel formats
+ for (int i = 0; i < num_pix_fmts; i++) {
+ for (int j = 0; j < FF_ARRAY_ELEMS(color_formats); j++) {
+ if (out_formats[i] == color_formats[j].color_format) {
+ probed_pix_fmts[i] = color_formats[j].pix_fmt;
+ break;
+ }
+ }
+ }
+ goto done;
+ }
+
+ num_pix_fmts = 0;
for (int i = 0; i < FF_ARRAY_ELEMS(color_formats); i++) {
if (color_formats[i].pix_fmt == AV_PIX_FMT_MEDIACODEC) {
// assumme AV_PIX_FMT_MEDIACODEC always works
@@ -589,6 +613,7 @@ static av_cold void mediacodec_init_static_data(FFCodec *ffcodec)
}
}
+done:
probed_pix_fmts[num_pix_fmts] = AV_PIX_FMT_NONE;
ffcodec->p.pix_fmts = probed_pix_fmts;
ff_AMediaCodec_delete(codec);
--
2.30.2