| Message ID | 20240528215607.62554-2-xaionaro@dx.center |
|---|---|
| State | New |
| Headers | show |
| Series | [FFmpeg-devel] avcodec/mediacodec: Add support of dynamic bitrate | expand |
| Context | Check | Description |
|---|---|---|
| yinshiyou/make_loongarch64 | success | Make finished |
| yinshiyou/make_fate_loongarch64 | success | Make fate finished |
| andriy/make_x86 | success | Make finished |
| andriy/make_fate_x86 | success | Make fate finished |
> 在 2024年5月29日，上午5:56，Dmitrii Okunev <xaionaro@gmail.com> 写道： > > MediaCodec supports parameter "video-bitrate" to change the bitrate > on fly. This commit adds capability to use it. > > It adds option -bitrate_ctrl_socket to the encoder which makes > the encoder to create an UNIX socket and listen for messages > to change the bitrate. Thank you for your contribution. However, libavcodec as a library has a clear define of margin, IO such as socket cannot be put into libavcodec. Reconfigure bitrate should be done via libavcodec API. > > An example of ffmpeg execution: > > ffmpeg -f flv -i rtmp://0.0.0.0:1935/live/myStream -c:v hevc_mediacodec -bitrate_ctrl_socket /run/bitrate.sock -b:v 8M -f flv rtmp://127.0.0.1:1935/live/reEncoded > > An example of changing the bitrate to 1000 Kbps: > > printf '%016X' 1000000 | xxd -r -p | socat -u STDIN UNIX:/run/bitrate.sock > > Signed-off-by: Dmitrii Okunev <xaionaro@gmail.com> > --- > libavcodec/Makefile | 2 +- > libavcodec/mediacodec_wrapper.c | 50 ++++++++ > libavcodec/mediacodec_wrapper.h | 12 ++ > libavcodec/mediacodecenc.c | 128 ++++++++++----------- > libavcodec/mediacodecenc.h | 97 ++++++++++++++++ > libavcodec/mediacodecenc_ctrl.c | 194 ++++++++++++++++++++++++++++++++ > libavcodec/mediacodecenc_ctrl.h | 50 ++++++++ > 7 files changed, 469 insertions(+), 64 deletions(-) > create mode 100644 libavcodec/mediacodecenc.h > create mode 100644 libavcodec/mediacodecenc_ctrl.c > create mode 100644 libavcodec/mediacodecenc_ctrl.h > > diff --git a/libavcodec/Makefile b/libavcodec/Makefile > index 2443d2c6fd..393877c8c6 100644 > --- a/libavcodec/Makefile > +++ b/libavcodec/Makefile > @@ -258,7 +258,7 @@ OBJS-$(CONFIG_AURA2_DECODER) += aura.o > OBJS-$(CONFIG_AV1_DECODER) += av1dec.o av1_parse.o > OBJS-$(CONFIG_AV1_CUVID_DECODER) += cuviddec.o > OBJS-$(CONFIG_AV1_MEDIACODEC_DECODER) += mediacodecdec.o > -OBJS-$(CONFIG_AV1_MEDIACODEC_ENCODER) += mediacodecenc.o > +OBJS-$(CONFIG_AV1_MEDIACODEC_ENCODER) += mediacodecenc.o mediacodecenc_ctrl.o > OBJS-$(CONFIG_AV1_NVENC_ENCODER) += nvenc_av1.o nvenc.o > OBJS-$(CONFIG_AV1_QSV_ENCODER) += qsvenc_av1.o > OBJS-$(CONFIG_AV1_VAAPI_ENCODER) += vaapi_encode_av1.o av1_levels.o > diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c > index 96c886666a..fe0e291cad 100644 > --- a/libavcodec/mediacodec_wrapper.c > +++ b/libavcodec/mediacodec_wrapper.c > @@ -174,6 +174,7 @@ struct JNIAMediaCodecFields { > jmethodID get_name_id; > > jmethodID configure_id; > + jmethodID set_parameters_id; > jmethodID start_id; > jmethodID flush_id; > jmethodID stop_id; > @@ -247,6 +248,9 @@ static const struct FFJniField jni_amediacodec_mapping[] = { > > { "android/media/MediaCodec", "setInputSurface", "(Landroid/view/Surface;)V", FF_JNI_METHOD, OFFSET(set_input_surface_id), 0 }, > { "android/media/MediaCodec", "signalEndOfInputStream", "()V", FF_JNI_METHOD, OFFSET(signal_end_of_input_stream_id), 0 }, > +#if __ANDROID_API__ >= 26 > + { "android/media/MediaCodec", "setParameters", "(Landroid/media/MediaFormat;)V", FF_JNI_METHOD, OFFSET(set_parameters_id), 1 }, > +#endif > > { "android/media/MediaCodec$BufferInfo", NULL, NULL, FF_JNI_CLASS, OFFSET(mediainfo_class), 1 }, > > @@ -1411,6 +1415,27 @@ fail: > return ret; > } > > +#if __ANDROID_API__ >= 26 > +static int mediacodec_jni_setParameters(FFAMediaCodec *ctx, > + const FFAMediaFormat *format_ctx) > +{ > + int ret = 0; > + JNIEnv *env = NULL; > + FFAMediaCodecJni *codec = (FFAMediaCodecJni *)ctx; > + const FFAMediaFormatJni *format = (FFAMediaFormatJni *)format_ctx; > + > + JNI_GET_ENV_OR_RETURN(env, codec, AVERROR_EXTERNAL); > + > + (*env)->CallVoidMethod(env, codec->object, codec->jfields.set_parameters_id, format->object); > + > + if (ff_jni_exception_check(env, 1, codec) < 0) { > + ret = AVERROR_EXTERNAL; > + } > + > + return ret; > +} > +#endif // __ANDROID_API__ >= 26 > + > static int mediacodec_jni_start(FFAMediaCodec* ctx) > { > int ret = 0; > @@ -1821,6 +1846,10 @@ static const FFAMediaCodec media_codec_jni = { > .getConfigureFlagEncode = mediacodec_jni_getConfigureFlagEncode, > .cleanOutputBuffers = mediacodec_jni_cleanOutputBuffers, > .signalEndOfInputStream = mediacodec_jni_signalEndOfInputStream, > + > +#if __ANDROID_API__ >= 26 > + .setParameters = mediacodec_jni_setParameters, > +#endif //__ANDROID_API__ >= 26 > }; > > typedef struct FFAMediaFormatNdk { > @@ -2178,6 +2207,23 @@ static int mediacodec_ndk_configure(FFAMediaCodec* ctx, > return 0; > } > > +#if __ANDROID_API__ >= 26 > +static int mediacodec_ndk_setParameters(FFAMediaCodec *ctx, > + const FFAMediaFormat *format_ctx) > +{ > + FFAMediaCodecNdk *codec = (FFAMediaCodecNdk *)ctx; > + FFAMediaFormatNdk *format = (FFAMediaFormatNdk *)format_ctx; > + > + int status = AMediaCodec_setParameters(codec->impl, format->impl); > + if (status != AMEDIA_OK) { > + av_log(codec, AV_LOG_ERROR, "codec setParameters failed, %d\n", status); > + return AVERROR_EXTERNAL; > + } > + > + return 0; > +} > +#endif //__ANDROID_API__ >= 26 > + > #define MEDIACODEC_NDK_WRAPPER(method) \ > static int mediacodec_ndk_ ## method(FFAMediaCodec* ctx) \ > { \ > @@ -2396,6 +2442,10 @@ static const FFAMediaCodec media_codec_ndk = { > .getConfigureFlagEncode = mediacodec_ndk_getConfigureFlagEncode, > .cleanOutputBuffers = mediacodec_ndk_cleanOutputBuffers, > .signalEndOfInputStream = mediacodec_ndk_signalEndOfInputStream, > + > +#if __ANDROID_API__ >= 26 > + .setParameters = mediacodec_ndk_setParameters, > +#endif //__ANDROID_API__ >= 26 > }; > > FFAMediaFormat *ff_AMediaFormat_new(int ndk) > diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h > index 11a4260497..9b5dbfb8dd 100644 > --- a/libavcodec/mediacodec_wrapper.h > +++ b/libavcodec/mediacodec_wrapper.h > @@ -219,6 +219,10 @@ struct FFAMediaCodec { > > // For encoder with FFANativeWindow as input. > int (*signalEndOfInputStream)(FFAMediaCodec *); > + > +#if __ANDROID_API__ >= 26 > + int (*setParameters)(FFAMediaCodec* codec, const FFAMediaFormat* format); > +#endif //__ANDROID_API__ >= 26 > }; > > static inline char *ff_AMediaCodec_getName(FFAMediaCodec *codec) > @@ -238,6 +242,14 @@ static inline int ff_AMediaCodec_configure(FFAMediaCodec *codec, > return codec->configure(codec, format, surface, crypto, flags); > } > > +#if __ANDROID_API__ >= 26 > +static inline int ff_AMediaCodec_setParameters(FFAMediaCodec *codec, > + const FFAMediaFormat *format) > +{ > + return codec->setParameters(codec, format); > +} > +#endif //__ANDROID_API__ >= 26 > + > static inline int ff_AMediaCodec_start(FFAMediaCodec* codec) > { > return codec->start(codec); > diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c > index bbf570e7be..b953457a5a 100644 > --- a/libavcodec/mediacodecenc.c > +++ b/libavcodec/mediacodecenc.c > @@ -29,76 +29,16 @@ > #include "libavutil/mem.h" > #include "libavutil/opt.h" > > -#include "avcodec.h" > -#include "bsf.h" > #include "codec_internal.h" > #include "encode.h" > #include "hwconfig.h" > #include "jni.h" > #include "mediacodec.h" > -#include "mediacodec_wrapper.h" > #include "mediacodecdec_common.h" > +#include "mediacodecenc_ctrl.h" > #include "profiles.h" > - > -#define INPUT_DEQUEUE_TIMEOUT_US 8000 > -#define OUTPUT_DEQUEUE_TIMEOUT_US 8000 > - > -enum BitrateMode { > - /* Constant quality mode */ > - BITRATE_MODE_CQ = 0, > - /* Variable bitrate mode */ > - BITRATE_MODE_VBR = 1, > - /* Constant bitrate mode */ > - BITRATE_MODE_CBR = 2, > - /* Constant bitrate mode with frame drops */ > - BITRATE_MODE_CBR_FD = 3, > -}; > - > -typedef struct MediaCodecEncContext { > - AVClass *avclass; > - FFAMediaCodec *codec; > - int use_ndk_codec; > - const char *name; > - FFANativeWindow *window; > - > - int fps; > - int width; > - int height; > - > - uint8_t *extradata; > - int extradata_size; > - int eof_sent; > - > - AVFrame *frame; > - AVBSFContext *bsf; > - > - int bitrate_mode; > - int level; > - int pts_as_dts; > - int extract_extradata; > -} MediaCodecEncContext; > - > -enum { > - COLOR_FormatYUV420Planar = 0x13, > - COLOR_FormatYUV420SemiPlanar = 0x15, > - COLOR_FormatSurface = 0x7F000789, > -}; > - > -static const struct { > - int color_format; > - enum AVPixelFormat pix_fmt; > -} color_formats[] = { > - { COLOR_FormatYUV420Planar, AV_PIX_FMT_YUV420P }, > - { COLOR_FormatYUV420SemiPlanar, AV_PIX_FMT_NV12 }, > - { COLOR_FormatSurface, AV_PIX_FMT_MEDIACODEC }, > -}; > - > -static const enum AVPixelFormat avc_pix_fmts[] = { > - AV_PIX_FMT_MEDIACODEC, > - AV_PIX_FMT_YUV420P, > - AV_PIX_FMT_NV12, > - AV_PIX_FMT_NONE > -}; > +#include "mediacodecenc.h" > +#include "mediacodecenc_ctrl.h" > > static void mediacodec_output_format(AVCodecContext *avctx) > { > @@ -307,6 +247,20 @@ static av_cold int mediacodec_init(AVCodecContext *avctx) > if (s->bitrate_mode == BITRATE_MODE_CQ && avctx->global_quality > 0) > ff_AMediaFormat_setInt32(format, "quality", avctx->global_quality); > } > + > + if (strlen(s->bitrate_ctrl_socket) != 0) { > +#if __ANDROID_API__ >= 26 > + int error = mediacodecenc_ctrl_init(avctx, format); > + if (error) { > + av_log(avctx, AV_LOG_ERROR, "unable to initialize the control socket '%s' for MediaCodec bitrate: errcode:%d, description:'%s'\n", s->bitrate_ctrl_socket, errno, strerror(errno)); > + goto bailout; > + } > +#else > + av_log(avctx, AV_LOG_ERROR, "ffmpeg was compiled against Android API version %d, but option -bitrate_ctrl_socket requires version 26\n", __ANDROID_API__); > + goto bailout; > +#endif //__ANDROID_API__ >= 26 > + } > + > // frame-rate and i-frame-interval are required to configure codec > if (avctx->framerate.num >= avctx->framerate.den && avctx->framerate.den > 0) { > s->fps = avctx->framerate.num / avctx->framerate.den; > @@ -385,6 +339,37 @@ bailout: > return ret; > } > > +#if __ANDROID_API__ >= 26 > +static void update_bitrate(AVCodecContext *avctx) { > + MediaCodecEncContext *s = avctx->priv_data; > + > + uint64_t bitrate_new = __atomic_load_n(&s->ctrl_ctx.bitrate_next, __ATOMIC_SEQ_CST); > + if (bitrate_new == s->ctrl_ctx.bitrate_cur) { > + return; > + } > + > + if (bitrate_new < 1) { > + return; > + } > + > + if (bitrate_new > UINT32_MAX) { > + av_log(avctx, AV_LOG_ERROR, "the requested bitrate %lu overflows a 32bit integer: %lu > %u\n", bitrate_new, bitrate_new, UINT32_MAX); > + return; > + } > + > + av_log(avctx, AV_LOG_DEBUG, "sending a message to update the bitrate through format %p using method %p\n", s->ctrl_ctx.out_format, s->ctrl_ctx.out_format->setInt32); > + > + ff_AMediaFormat_setInt32(s->ctrl_ctx.out_format, (char *)"video-bitrate", bitrate_new); > + if (ff_AMediaCodec_setParameters(s->codec, s->ctrl_ctx.out_format)) { > + av_log(avctx, AV_LOG_ERROR, "unable to set the bitrate to %lu\n", bitrate_new); > + return; > + } > + > + av_log(avctx, AV_LOG_INFO, "changed the bitrate: %lu -> %lu\n", s->ctrl_ctx.bitrate_cur, bitrate_new); > + __atomic_store_n(&s->ctrl_ctx.bitrate_cur, bitrate_new, __ATOMIC_SEQ_CST); > +} > +#endif //__ANDROID_API__ >= 26 > + > static int mediacodec_receive(AVCodecContext *avctx, AVPacket *pkt) > { > MediaCodecEncContext *s = avctx->priv_data; > @@ -395,6 +380,7 @@ static int mediacodec_receive(AVCodecContext *avctx, AVPacket *pkt) > int ret; > int extradata_size = 0; > int64_t timeout_us = s->eof_sent ? OUTPUT_DEQUEUE_TIMEOUT_US : 0; > + > ssize_t index = ff_AMediaCodec_dequeueOutputBuffer(codec, &out_info, timeout_us); > > if (ff_AMediaCodec_infoTryAgainLater(codec, index)) > @@ -563,6 +549,13 @@ static int mediacodec_encode(AVCodecContext *avctx, AVPacket *pkt) > return 0; > } > > +#if __ANDROID_API__ >= 26 > + if (strlen(s->bitrate_ctrl_socket) != 0 && s->ctrl_ctx.bitrate_next != s->ctrl_ctx.bitrate_cur) { > + av_log(avctx, AV_LOG_TRACE, "calling update_bitrate\n"); > + update_bitrate(avctx); > + } > +#endif //__ANDROID_API__ >= 26 > + > if (ret < 0 && ret != AVERROR(EAGAIN)) > return ret; > > @@ -689,6 +682,13 @@ bailout: > static av_cold int mediacodec_close(AVCodecContext *avctx) > { > MediaCodecEncContext *s = avctx->priv_data; > + > +#if __ANDROID_API__ >= 26 > + if (strlen(s->bitrate_ctrl_socket) != 0) { > + mediacodecenc_ctrl_deinit(avctx); > + } > +#endif //__ANDROID_API__ >= 26 > + > if (s->codec) { > ff_AMediaCodec_stop(s->codec); > ff_AMediaCodec_delete(s->codec); > @@ -737,6 +737,8 @@ static const AVCodecHWConfigInternal *const mediacodec_hw_configs[] = { > OFFSET(name), AV_OPT_TYPE_STRING, {0}, 0, 0, VE }, \ > { "bitrate_mode", "Bitrate control method", \ > OFFSET(bitrate_mode), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, VE, .unit = "bitrate_mode" }, \ > + { "bitrate_ctrl_socket", "Enable (and set path to) a control UNIX socket that receives uint64_t big-endian messages that dynamically override the bitrate; the value is given in bits per second.", \ > + OFFSET(bitrate_ctrl_socket), AV_OPT_TYPE_STRING, {.str = ""}, 0, 0, VE }, \ > { "cq", "Constant quality mode", \ > 0, AV_OPT_TYPE_CONST, {.i64 = BITRATE_MODE_CQ}, 0, 0, VE, .unit = "bitrate_mode" }, \ > { "vbr", "Variable bitrate mode", \ > diff --git a/libavcodec/mediacodecenc.h b/libavcodec/mediacodecenc.h > new file mode 100644 > index 0000000000..8915c99f54 > --- /dev/null > +++ b/libavcodec/mediacodecenc.h > @@ -0,0 +1,97 @@ > +/* > + * Android MediaCodec encoders > + * > + * Copyright (c) 2022 Zhao Zhili <zhilizhao@tencent.com> > + * Copyright (c) 2024 Dmitrii Okunev <xaionaro@gmail.com> > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#ifndef AVCODEC_MEDIACODECENC_H > +#define AVCODEC_MEDIACODECENC_H > + > +#include "avcodec.h" > +#include "bsf.h" > +#include "mediacodec_wrapper.h" > +#include "mediacodecenc_ctrl.h" > + > +#define INPUT_DEQUEUE_TIMEOUT_US 8000 > +#define OUTPUT_DEQUEUE_TIMEOUT_US 8000 > + > +enum BitrateMode { > + /* Constant quality mode */ > + BITRATE_MODE_CQ = 0, > + /* Variable bitrate mode */ > + BITRATE_MODE_VBR = 1, > + /* Constant bitrate mode */ > + BITRATE_MODE_CBR = 2, > + /* Constant bitrate mode with frame drops */ > + BITRATE_MODE_CBR_FD = 3, > +}; > + > +typedef struct MediaCodecEncContext { > + AVClass *avclass; > + FFAMediaCodec *codec; > + int use_ndk_codec; > + const char *name; > + FFANativeWindow *window; > + > + int fps; > + int width; > + int height; > + > + uint8_t *extradata; > + int extradata_size; > + int eof_sent; > + > + AVFrame *frame; > + AVBSFContext *bsf; > + > + int bitrate_mode; > + char *bitrate_ctrl_socket; > + int level; > + int pts_as_dts; > + int extract_extradata; > + > +#if __ANDROID_API__ >= 26 > + MediaCodecEncControlContext ctrl_ctx; > +#endif //__ANDROID_API__ >= 26 > +} MediaCodecEncContext; > + > +enum { > + COLOR_FormatYUV420Planar = 0x13, > + COLOR_FormatYUV420SemiPlanar = 0x15, > + COLOR_FormatSurface = 0x7F000789, > +}; > + > +static const struct { > + int color_format; > + enum AVPixelFormat pix_fmt; > +} color_formats[] = { > + { COLOR_FormatYUV420Planar, AV_PIX_FMT_YUV420P }, > + { COLOR_FormatYUV420SemiPlanar, AV_PIX_FMT_NV12 }, > + { COLOR_FormatSurface, AV_PIX_FMT_MEDIACODEC }, > +}; > + > +static const enum AVPixelFormat avc_pix_fmts[] = { > + AV_PIX_FMT_MEDIACODEC, > + AV_PIX_FMT_YUV420P, > + AV_PIX_FMT_NV12, > + AV_PIX_FMT_NONE > +}; > + > +#endif /* AVCODEC_MEDIACODECENC_H */ > diff --git a/libavcodec/mediacodecenc_ctrl.c b/libavcodec/mediacodecenc_ctrl.c > new file mode 100644 > index 0000000000..a732594ecd > --- /dev/null > +++ b/libavcodec/mediacodecenc_ctrl.c > @@ -0,0 +1,194 @@ > +/* > + * Android MediaCodec encoders > + * > + * Copyright (c) 2024 Dmitrii Okunev <xaionaro@gmail.com> > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#if __ANDROID_API__ >= 26 > + > +#include <endian.h> > +#include <stdatomic.h> > +#include <stdio.h> > +#include <stdlib.h> > +#include <stdint.h> > +#include <sys/socket.h> > +#include <sys/un.h> > +#include <unistd.h> > +#include <pthread.h> > + > +#include "mediacodecenc_ctrl.h" > +#include "mediacodecenc.h" > + > +typedef struct mediacodecenc_ctrl_handle_connection_context { > + int client_sock_fd; > + AVCodecContext *avctx; > +} mediacodecenc_ctrl_handle_connection_context_t; > + > +static void *mediacodecenc_ctrl_handle_connection(void *arg) { > + mediacodecenc_ctrl_handle_connection_context_t *hctx = arg; > + AVCodecContext *avctx = hctx->avctx; > + MediaCodecEncContext *s = avctx->priv_data; > + > + while (1) { > + uint64_t received_value, bitrate_old, bitrate_new; > + ssize_t num_bytes = recv(hctx->client_sock_fd, &received_value, sizeof(received_value), MSG_CMSG_CLOEXEC); > + if (num_bytes <= 0) { > + switch (errno) { > + case 0: > + case ECONNRESET: > + case EPIPE: > + case EBADF: > + av_log(avctx, AV_LOG_INFO, "The MediaCodec control connection was closed: errno:%d: %s\n", errno, strerror(errno)); > + break; > + default: > + av_log(avctx, AV_LOG_ERROR, "A failure of a MediaCodec control connection: errno:%d: %s\n", errno, strerror(errno)); > + } > + break; > + } > + > + bitrate_new = be64toh(received_value); > + > + av_log(avctx, AV_LOG_TRACE, "Received a message to change the bitrate to %lu.\n", bitrate_new); > + > + bitrate_old = __atomic_load_n(&s->ctrl_ctx.bitrate_cur, __ATOMIC_SEQ_CST); > + if (bitrate_new == bitrate_old) { > + av_log(avctx, AV_LOG_INFO, "Received a message to change the bitrate, but the bitrate is already %lu, not changing.\n", bitrate_old); > + continue; > + } > + > + av_log(avctx, AV_LOG_INFO, "Received a message to change the bitrate from %lu to %lu.\n", bitrate_old, bitrate_new); > + __atomic_store_n(&s->ctrl_ctx.bitrate_next, bitrate_new, __ATOMIC_SEQ_CST); > + } > + > + close(hctx->client_sock_fd); > + free(hctx); > + pthread_exit(NULL); > +} > + > +static void *mediacodecenc_ctrl_handle_listener(void *_avctx) { > + AVCodecContext *avctx = _avctx; > + MediaCodecEncContext *s = avctx->priv_data; > + > + if (listen(s->ctrl_ctx.server_sock_fd, 5) < 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to start listening socket '%s': errno:%d: %s\n", s->ctrl_ctx.server_sock_path, errno, strerror(errno)); > + goto mediacodecenc_ctrl_handle_listener_end; > + } > + > + av_log(avctx, AV_LOG_TRACE, "Listening socket '%s'\n", s->ctrl_ctx.server_sock_path); > + > + s->ctrl_ctx.listener_status = mcls_running; > + while (s->ctrl_ctx.listener_status == mcls_running) { > + pthread_t client_thread_id; > + int pthread_error; > + mediacodecenc_ctrl_handle_connection_context_t *hctx; > + > + int client_sock_fd = accept(s->ctrl_ctx.server_sock_fd, NULL, NULL); > + if (client_sock_fd < 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to accept a connection to socket '%s': code:%d: description:%s\n", s->ctrl_ctx.server_sock_path, errno, strerror(errno)); > + continue; > + } > + > + av_log(avctx, AV_LOG_TRACE, "Accepted a connection to '%s'\n", s->ctrl_ctx.server_sock_path); > + > + hctx = malloc(sizeof(mediacodecenc_ctrl_handle_connection_context_t)); > + hctx->client_sock_fd = client_sock_fd; > + hctx->avctx = avctx; > + > + pthread_error = pthread_create(&client_thread_id, NULL, mediacodecenc_ctrl_handle_connection, hctx); > + if (pthread_error != 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to accept a connection to socket '%s': code:%d: description:%s\n", s->ctrl_ctx.server_sock_path, pthread_error, strerror(pthread_error)); > + free(hctx); > + close(client_sock_fd); > + continue; > + } > + > + // mediacodecenc_ctrl_handle_connection takes ownership of hctx and frees it when finished, > + // so we are only detaching here and that's it. > + pthread_error = pthread_detach(client_thread_id); > + if (pthread_error != 0) { > + av_log(avctx, AV_LOG_ERROR, "Unable to detach the client handler thread: code:%d: description:%s\n", pthread_error, strerror(pthread_error)); > + } > + } > + > +mediacodecenc_ctrl_handle_listener_end: > + close(s->ctrl_ctx.server_sock_fd); > + unlink(s->ctrl_ctx.server_sock_path); > + s->ctrl_ctx.listener_status = mcls_stopped; > + pthread_exit(NULL); > +} > + > +int mediacodecenc_ctrl_init(AVCodecContext *avctx, FFAMediaFormat *format) { > + MediaCodecEncContext *s = avctx->priv_data; > + char *socket_path = s->bitrate_ctrl_socket; > + int server_sock_fd; > + struct sockaddr_un server_addr; > + int pthread_error; > + > + s->ctrl_ctx.bitrate_cur = avctx->bit_rate; > + s->ctrl_ctx.bitrate_next = s->ctrl_ctx.bitrate_cur; > + s->ctrl_ctx.out_format = format; > + > + server_sock_fd = socket(AF_UNIX, SOCK_STREAM, 0); > + if (server_sock_fd < 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create socket '%s': errno:%d: %s\n", socket_path, errno, strerror(errno)); > + return errno; > + } > + > + memset(&server_addr, 0, sizeof(struct sockaddr_un)); > + server_addr.sun_family = AF_UNIX; > + strncpy(server_addr.sun_path, socket_path, sizeof(server_addr.sun_path) - 1); > + > + if (bind(server_sock_fd, (struct sockaddr *)&server_addr, sizeof(struct sockaddr_un)) < 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to bind to the socket '%s': errno:%d: %s\n", socket_path, errno, strerror(errno)); > + return errno; > + } > + s->ctrl_ctx.server_sock_path = socket_path; > + s->ctrl_ctx.server_sock_fd = server_sock_fd; > + > + pthread_error = pthread_create(&s->ctrl_ctx.listener_thread_id, NULL, mediacodecenc_ctrl_handle_listener, avctx); > + if (pthread_error != 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create a thread to listen the socket '%s': errno:%d: %s\n", socket_path, pthread_error, strerror(pthread_error)); > + return pthread_error; > + } > + > + return 0; > +} > + > +int mediacodecenc_ctrl_deinit(AVCodecContext *avctx) { > + MediaCodecEncContext *s = avctx->priv_data; > + void *retval; > + int pthread_error; > + if (s->ctrl_ctx.listener_thread_id != 0) { > + return 0; > + } > + > + s->ctrl_ctx.listener_status = mcls_stopping; > + close(s->ctrl_ctx.server_sock_fd); > + > + pthread_error = pthread_join(s->ctrl_ctx.listener_thread_id, &retval); > + if (pthread_error != 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to detach from the listener thread of the socket '%s': errno:%d: %s\n", s->ctrl_ctx.server_sock_path, pthread_error, strerror(pthread_error)); > + return pthread_error; > + } > + > + ff_AMediaFormat_delete(s->ctrl_ctx.out_format); > + return 0; > +} > + > +#endif //__ANDROID_API__ >= 26 > diff --git a/libavcodec/mediacodecenc_ctrl.h b/libavcodec/mediacodecenc_ctrl.h > new file mode 100644 > index 0000000000..a5df203639 > --- /dev/null > +++ b/libavcodec/mediacodecenc_ctrl.h > @@ -0,0 +1,50 @@ > +/* > + * Android MediaCodec encoders > + * > + * Copyright (c) 2024 Dmitrii Okunev <xaionaro@gmail.com> > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#ifndef AVCODEC_MEDIACODECENC_CTRL_H > +#define AVCODEC_MEDIACODECENC_CTRL_H > +#if __ANDROID_API__ >= 26 > + > +#include "avcodec.h" > +#include "mediacodec_wrapper.h" > + > +typedef enum mediacodecenc_ctrl_listener_status { > + mcls_stopped = 0, > + mcls_running = 1, > + mcls_stopping = 2, > +} mediacodecenc_ctrl_listener_status_t; > + > +typedef struct mediacodecenc_ctrl_ctx { > + char *server_sock_path; > + int server_sock_fd; > + pthread_t listener_thread_id; > + mediacodecenc_ctrl_listener_status_t listener_status; > + uint64_t bitrate_cur; > + uint64_t bitrate_next; > + FFAMediaFormat *out_format; > +} MediaCodecEncControlContext; > + > +extern int mediacodecenc_ctrl_init(AVCodecContext *avctx, FFAMediaFormat *format); > +extern int mediacodecenc_ctrl_deinit(AVCodecContext *avctx); > + > +#endif //__ANDROID_API__ >= 26 > +#endif /* AVCODEC_MEDIACODECENC_CTRL_H */ > -- > 2.43.0 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email
Thanks for the response. On Wed, 2024-05-29 at 12:19 +0800, Zhao Zhili wrote: > > 在 2024年5月29日，上午5:56，Dmitrii Okunev <xaionaro@gmail.com> 写道： > > > > MediaCodec supports parameter "video-bitrate" to change the bitrate > > on fly. This commit adds capability to use it. > > > > It adds option -bitrate_ctrl_socket to the encoder which makes > > the encoder to create an UNIX socket and listen for messages > > to change the bitrate. > > Thank you for your contribution. However, libavcodec as a library has a clear define of margin, IO such as socket cannot be put into libavcodec. Reconfigure bitrate should be done via libavcodec API. This was the last resort, I tried to find another solution before posting this patch. But of course I'll gladly adapt the solution: could you give examples/pointers on how to change values via libavcodec API on fly in ffmpeg? Best regards, Dmitrii.
diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 2443d2c6fd..393877c8c6 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -258,7 +258,7 @@ OBJS-$(CONFIG_AURA2_DECODER) += aura.o OBJS-$(CONFIG_AV1_DECODER) += av1dec.o av1_parse.o OBJS-$(CONFIG_AV1_CUVID_DECODER) += cuviddec.o OBJS-$(CONFIG_AV1_MEDIACODEC_DECODER) += mediacodecdec.o -OBJS-$(CONFIG_AV1_MEDIACODEC_ENCODER) += mediacodecenc.o +OBJS-$(CONFIG_AV1_MEDIACODEC_ENCODER) += mediacodecenc.o mediacodecenc_ctrl.o OBJS-$(CONFIG_AV1_NVENC_ENCODER) += nvenc_av1.o nvenc.o OBJS-$(CONFIG_AV1_QSV_ENCODER) += qsvenc_av1.o OBJS-$(CONFIG_AV1_VAAPI_ENCODER) += vaapi_encode_av1.o av1_levels.o diff --git a/libavcodec/mediacodec_wrapper.c b/libavcodec/mediacodec_wrapper.c index 96c886666a..fe0e291cad 100644 --- a/libavcodec/mediacodec_wrapper.c +++ b/libavcodec/mediacodec_wrapper.c @@ -174,6 +174,7 @@ struct JNIAMediaCodecFields { jmethodID get_name_id; jmethodID configure_id; + jmethodID set_parameters_id; jmethodID start_id; jmethodID flush_id; jmethodID stop_id; @@ -247,6 +248,9 @@ static const struct FFJniField jni_amediacodec_mapping[] = { { "android/media/MediaCodec", "setInputSurface", "(Landroid/view/Surface;)V", FF_JNI_METHOD, OFFSET(set_input_surface_id), 0 }, { "android/media/MediaCodec", "signalEndOfInputStream", "()V", FF_JNI_METHOD, OFFSET(signal_end_of_input_stream_id), 0 }, +#if __ANDROID_API__ >= 26 + { "android/media/MediaCodec", "setParameters", "(Landroid/media/MediaFormat;)V", FF_JNI_METHOD, OFFSET(set_parameters_id), 1 }, +#endif { "android/media/MediaCodec$BufferInfo", NULL, NULL, FF_JNI_CLASS, OFFSET(mediainfo_class), 1 }, @@ -1411,6 +1415,27 @@ fail: return ret; } +#if __ANDROID_API__ >= 26 +static int mediacodec_jni_setParameters(FFAMediaCodec *ctx, + const FFAMediaFormat *format_ctx) +{ + int ret = 0; + JNIEnv *env = NULL; + FFAMediaCodecJni *codec = (FFAMediaCodecJni *)ctx; + const FFAMediaFormatJni *format = (FFAMediaFormatJni *)format_ctx; + + JNI_GET_ENV_OR_RETURN(env, codec, AVERROR_EXTERNAL); + + (*env)->CallVoidMethod(env, codec->object, codec->jfields.set_parameters_id, format->object); + + if (ff_jni_exception_check(env, 1, codec) < 0) { + ret = AVERROR_EXTERNAL; + } + + return ret; +} +#endif // __ANDROID_API__ >= 26 + static int mediacodec_jni_start(FFAMediaCodec* ctx) { int ret = 0; @@ -1821,6 +1846,10 @@ static const FFAMediaCodec media_codec_jni = { .getConfigureFlagEncode = mediacodec_jni_getConfigureFlagEncode, .cleanOutputBuffers = mediacodec_jni_cleanOutputBuffers, .signalEndOfInputStream = mediacodec_jni_signalEndOfInputStream, + +#if __ANDROID_API__ >= 26 + .setParameters = mediacodec_jni_setParameters, +#endif //__ANDROID_API__ >= 26 }; typedef struct FFAMediaFormatNdk { @@ -2178,6 +2207,23 @@ static int mediacodec_ndk_configure(FFAMediaCodec* ctx, return 0; } +#if __ANDROID_API__ >= 26 +static int mediacodec_ndk_setParameters(FFAMediaCodec *ctx, + const FFAMediaFormat *format_ctx) +{ + FFAMediaCodecNdk *codec = (FFAMediaCodecNdk *)ctx; + FFAMediaFormatNdk *format = (FFAMediaFormatNdk *)format_ctx; + + int status = AMediaCodec_setParameters(codec->impl, format->impl); + if (status != AMEDIA_OK) { + av_log(codec, AV_LOG_ERROR, "codec setParameters failed, %d\n", status); + return AVERROR_EXTERNAL; + } + + return 0; +} +#endif //__ANDROID_API__ >= 26 + #define MEDIACODEC_NDK_WRAPPER(method) \ static int mediacodec_ndk_ ## method(FFAMediaCodec* ctx) \ { \ @@ -2396,6 +2442,10 @@ static const FFAMediaCodec media_codec_ndk = { .getConfigureFlagEncode = mediacodec_ndk_getConfigureFlagEncode, .cleanOutputBuffers = mediacodec_ndk_cleanOutputBuffers, .signalEndOfInputStream = mediacodec_ndk_signalEndOfInputStream, + +#if __ANDROID_API__ >= 26 + .setParameters = mediacodec_ndk_setParameters, +#endif //__ANDROID_API__ >= 26 }; FFAMediaFormat *ff_AMediaFormat_new(int ndk) diff --git a/libavcodec/mediacodec_wrapper.h b/libavcodec/mediacodec_wrapper.h index 11a4260497..9b5dbfb8dd 100644 --- a/libavcodec/mediacodec_wrapper.h +++ b/libavcodec/mediacodec_wrapper.h @@ -219,6 +219,10 @@ struct FFAMediaCodec { // For encoder with FFANativeWindow as input. int (*signalEndOfInputStream)(FFAMediaCodec *); + +#if __ANDROID_API__ >= 26 + int (*setParameters)(FFAMediaCodec* codec, const FFAMediaFormat* format); +#endif //__ANDROID_API__ >= 26 }; static inline char *ff_AMediaCodec_getName(FFAMediaCodec *codec) @@ -238,6 +242,14 @@ static inline int ff_AMediaCodec_configure(FFAMediaCodec *codec, return codec->configure(codec, format, surface, crypto, flags); } +#if __ANDROID_API__ >= 26 +static inline int ff_AMediaCodec_setParameters(FFAMediaCodec *codec, + const FFAMediaFormat *format) +{ + return codec->setParameters(codec, format); +} +#endif //__ANDROID_API__ >= 26 + static inline int ff_AMediaCodec_start(FFAMediaCodec* codec) { return codec->start(codec); diff --git a/libavcodec/mediacodecenc.c b/libavcodec/mediacodecenc.c index bbf570e7be..b953457a5a 100644 --- a/libavcodec/mediacodecenc.c +++ b/libavcodec/mediacodecenc.c @@ -29,76 +29,16 @@ #include "libavutil/mem.h" #include "libavutil/opt.h" -#include "avcodec.h" -#include "bsf.h" #include "codec_internal.h" #include "encode.h" #include "hwconfig.h" #include "jni.h" #include "mediacodec.h" -#include "mediacodec_wrapper.h" #include "mediacodecdec_common.h" +#include "mediacodecenc_ctrl.h" #include "profiles.h" - -#define INPUT_DEQUEUE_TIMEOUT_US 8000 -#define OUTPUT_DEQUEUE_TIMEOUT_US 8000 - -enum BitrateMode { - /* Constant quality mode */ - BITRATE_MODE_CQ = 0, - /* Variable bitrate mode */ - BITRATE_MODE_VBR = 1, - /* Constant bitrate mode */ - BITRATE_MODE_CBR = 2, - /* Constant bitrate mode with frame drops */ - BITRATE_MODE_CBR_FD = 3, -}; - -typedef struct MediaCodecEncContext { - AVClass *avclass; - FFAMediaCodec *codec; - int use_ndk_codec; - const char *name; - FFANativeWindow *window; - - int fps; - int width; - int height; - - uint8_t *extradata; - int extradata_size; - int eof_sent; - - AVFrame *frame; - AVBSFContext *bsf; - - int bitrate_mode; - int level; - int pts_as_dts; - int extract_extradata; -} MediaCodecEncContext; - -enum { - COLOR_FormatYUV420Planar = 0x13, - COLOR_FormatYUV420SemiPlanar = 0x15, - COLOR_FormatSurface = 0x7F000789, -}; - -static const struct { - int color_format; - enum AVPixelFormat pix_fmt; -} color_formats[] = { - { COLOR_FormatYUV420Planar, AV_PIX_FMT_YUV420P }, - { COLOR_FormatYUV420SemiPlanar, AV_PIX_FMT_NV12 }, - { COLOR_FormatSurface, AV_PIX_FMT_MEDIACODEC }, -}; - -static const enum AVPixelFormat avc_pix_fmts[] = { - AV_PIX_FMT_MEDIACODEC, - AV_PIX_FMT_YUV420P, - AV_PIX_FMT_NV12, - AV_PIX_FMT_NONE -}; +#include "mediacodecenc.h" +#include "mediacodecenc_ctrl.h" static void mediacodec_output_format(AVCodecContext *avctx) { @@ -307,6 +247,20 @@ static av_cold int mediacodec_init(AVCodecContext *avctx) if (s->bitrate_mode == BITRATE_MODE_CQ && avctx->global_quality > 0) ff_AMediaFormat_setInt32(format, "quality", avctx->global_quality); } + + if (strlen(s->bitrate_ctrl_socket) != 0) { +#if __ANDROID_API__ >= 26 + int error = mediacodecenc_ctrl_init(avctx, format); + if (error) { + av_log(avctx, AV_LOG_ERROR, "unable to initialize the control socket '%s' for MediaCodec bitrate: errcode:%d, description:'%s'\n", s->bitrate_ctrl_socket, errno, strerror(errno)); + goto bailout; + } +#else + av_log(avctx, AV_LOG_ERROR, "ffmpeg was compiled against Android API version %d, but option -bitrate_ctrl_socket requires version 26\n", __ANDROID_API__); + goto bailout; +#endif //__ANDROID_API__ >= 26 + } + // frame-rate and i-frame-interval are required to configure codec if (avctx->framerate.num >= avctx->framerate.den && avctx->framerate.den > 0) { s->fps = avctx->framerate.num / avctx->framerate.den; @@ -385,6 +339,37 @@ bailout: return ret; } +#if __ANDROID_API__ >= 26 +static void update_bitrate(AVCodecContext *avctx) { + MediaCodecEncContext *s = avctx->priv_data; + + uint64_t bitrate_new = __atomic_load_n(&s->ctrl_ctx.bitrate_next, __ATOMIC_SEQ_CST); + if (bitrate_new == s->ctrl_ctx.bitrate_cur) { + return; + } + + if (bitrate_new < 1) { + return; + } + + if (bitrate_new > UINT32_MAX) { + av_log(avctx, AV_LOG_ERROR, "the requested bitrate %lu overflows a 32bit integer: %lu > %u\n", bitrate_new, bitrate_new, UINT32_MAX); + return; + } + + av_log(avctx, AV_LOG_DEBUG, "sending a message to update the bitrate through format %p using method %p\n", s->ctrl_ctx.out_format, s->ctrl_ctx.out_format->setInt32); + + ff_AMediaFormat_setInt32(s->ctrl_ctx.out_format, (char *)"video-bitrate", bitrate_new); + if (ff_AMediaCodec_setParameters(s->codec, s->ctrl_ctx.out_format)) { + av_log(avctx, AV_LOG_ERROR, "unable to set the bitrate to %lu\n", bitrate_new); + return; + } + + av_log(avctx, AV_LOG_INFO, "changed the bitrate: %lu -> %lu\n", s->ctrl_ctx.bitrate_cur, bitrate_new); + __atomic_store_n(&s->ctrl_ctx.bitrate_cur, bitrate_new, __ATOMIC_SEQ_CST); +} +#endif //__ANDROID_API__ >= 26 + static int mediacodec_receive(AVCodecContext *avctx, AVPacket *pkt) { MediaCodecEncContext *s = avctx->priv_data; @@ -395,6 +380,7 @@ static int mediacodec_receive(AVCodecContext *avctx, AVPacket *pkt) int ret; int extradata_size = 0; int64_t timeout_us = s->eof_sent ? OUTPUT_DEQUEUE_TIMEOUT_US : 0; + ssize_t index = ff_AMediaCodec_dequeueOutputBuffer(codec, &out_info, timeout_us); if (ff_AMediaCodec_infoTryAgainLater(codec, index)) @@ -563,6 +549,13 @@ static int mediacodec_encode(AVCodecContext *avctx, AVPacket *pkt) return 0; } +#if __ANDROID_API__ >= 26 + if (strlen(s->bitrate_ctrl_socket) != 0 && s->ctrl_ctx.bitrate_next != s->ctrl_ctx.bitrate_cur) { + av_log(avctx, AV_LOG_TRACE, "calling update_bitrate\n"); + update_bitrate(avctx); + } +#endif //__ANDROID_API__ >= 26 + if (ret < 0 && ret != AVERROR(EAGAIN)) return ret; @@ -689,6 +682,13 @@ bailout: static av_cold int mediacodec_close(AVCodecContext *avctx) { MediaCodecEncContext *s = avctx->priv_data; + +#if __ANDROID_API__ >= 26 + if (strlen(s->bitrate_ctrl_socket) != 0) { + mediacodecenc_ctrl_deinit(avctx); + } +#endif //__ANDROID_API__ >= 26 + if (s->codec) { ff_AMediaCodec_stop(s->codec); ff_AMediaCodec_delete(s->codec); @@ -737,6 +737,8 @@ static const AVCodecHWConfigInternal *const mediacodec_hw_configs[] = { OFFSET(name), AV_OPT_TYPE_STRING, {0}, 0, 0, VE }, \ { "bitrate_mode", "Bitrate control method", \ OFFSET(bitrate_mode), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, VE, .unit = "bitrate_mode" }, \ + { "bitrate_ctrl_socket", "Enable (and set path to) a control UNIX socket that receives uint64_t big-endian messages that dynamically override the bitrate; the value is given in bits per second.", \ + OFFSET(bitrate_ctrl_socket), AV_OPT_TYPE_STRING, {.str = ""}, 0, 0, VE }, \ { "cq", "Constant quality mode", \ 0, AV_OPT_TYPE_CONST, {.i64 = BITRATE_MODE_CQ}, 0, 0, VE, .unit = "bitrate_mode" }, \ { "vbr", "Variable bitrate mode", \ diff --git a/libavcodec/mediacodecenc.h b/libavcodec/mediacodecenc.h new file mode 100644 index 0000000000..8915c99f54 --- /dev/null +++ b/libavcodec/mediacodecenc.h @@ -0,0 +1,97 @@ +/* + * Android MediaCodec encoders + * + * Copyright (c) 2022 Zhao Zhili <zhilizhao@tencent.com> + * Copyright (c) 2024 Dmitrii Okunev <xaionaro@gmail.com> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_MEDIACODECENC_H +#define AVCODEC_MEDIACODECENC_H + +#include "avcodec.h" +#include "bsf.h" +#include "mediacodec_wrapper.h" +#include "mediacodecenc_ctrl.h" + +#define INPUT_DEQUEUE_TIMEOUT_US 8000 +#define OUTPUT_DEQUEUE_TIMEOUT_US 8000 + +enum BitrateMode { + /* Constant quality mode */ + BITRATE_MODE_CQ = 0, + /* Variable bitrate mode */ + BITRATE_MODE_VBR = 1, + /* Constant bitrate mode */ + BITRATE_MODE_CBR = 2, + /* Constant bitrate mode with frame drops */ + BITRATE_MODE_CBR_FD = 3, +}; + +typedef struct MediaCodecEncContext { + AVClass *avclass; + FFAMediaCodec *codec; + int use_ndk_codec; + const char *name; + FFANativeWindow *window; + + int fps; + int width; + int height; + + uint8_t *extradata; + int extradata_size; + int eof_sent; + + AVFrame *frame; + AVBSFContext *bsf; + + int bitrate_mode; + char *bitrate_ctrl_socket; + int level; + int pts_as_dts; + int extract_extradata; + +#if __ANDROID_API__ >= 26 + MediaCodecEncControlContext ctrl_ctx; +#endif //__ANDROID_API__ >= 26 +} MediaCodecEncContext; + +enum { + COLOR_FormatYUV420Planar = 0x13, + COLOR_FormatYUV420SemiPlanar = 0x15, + COLOR_FormatSurface = 0x7F000789, +}; + +static const struct { + int color_format; + enum AVPixelFormat pix_fmt; +} color_formats[] = { + { COLOR_FormatYUV420Planar, AV_PIX_FMT_YUV420P }, + { COLOR_FormatYUV420SemiPlanar, AV_PIX_FMT_NV12 }, + { COLOR_FormatSurface, AV_PIX_FMT_MEDIACODEC }, +}; + +static const enum AVPixelFormat avc_pix_fmts[] = { + AV_PIX_FMT_MEDIACODEC, + AV_PIX_FMT_YUV420P, + AV_PIX_FMT_NV12, + AV_PIX_FMT_NONE +}; + +#endif /* AVCODEC_MEDIACODECENC_H */ diff --git a/libavcodec/mediacodecenc_ctrl.c b/libavcodec/mediacodecenc_ctrl.c new file mode 100644 index 0000000000..a732594ecd --- /dev/null +++ b/libavcodec/mediacodecenc_ctrl.c @@ -0,0 +1,194 @@ +/* + * Android MediaCodec encoders + * + * Copyright (c) 2024 Dmitrii Okunev <xaionaro@gmail.com> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#if __ANDROID_API__ >= 26 + +#include <endian.h> +#include <stdatomic.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdint.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <unistd.h> +#include <pthread.h> + +#include "mediacodecenc_ctrl.h" +#include "mediacodecenc.h" + +typedef struct mediacodecenc_ctrl_handle_connection_context { + int client_sock_fd; + AVCodecContext *avctx; +} mediacodecenc_ctrl_handle_connection_context_t; + +static void *mediacodecenc_ctrl_handle_connection(void *arg) { + mediacodecenc_ctrl_handle_connection_context_t *hctx = arg; + AVCodecContext *avctx = hctx->avctx; + MediaCodecEncContext *s = avctx->priv_data; + + while (1) { + uint64_t received_value, bitrate_old, bitrate_new; + ssize_t num_bytes = recv(hctx->client_sock_fd, &received_value, sizeof(received_value), MSG_CMSG_CLOEXEC); + if (num_bytes <= 0) { + switch (errno) { + case 0: + case ECONNRESET: + case EPIPE: + case EBADF: + av_log(avctx, AV_LOG_INFO, "The MediaCodec control connection was closed: errno:%d: %s\n", errno, strerror(errno)); + break; + default: + av_log(avctx, AV_LOG_ERROR, "A failure of a MediaCodec control connection: errno:%d: %s\n", errno, strerror(errno)); + } + break; + } + + bitrate_new = be64toh(received_value); + + av_log(avctx, AV_LOG_TRACE, "Received a message to change the bitrate to %lu.\n", bitrate_new); + + bitrate_old = __atomic_load_n(&s->ctrl_ctx.bitrate_cur, __ATOMIC_SEQ_CST); + if (bitrate_new == bitrate_old) { + av_log(avctx, AV_LOG_INFO, "Received a message to change the bitrate, but the bitrate is already %lu, not changing.\n", bitrate_old); + continue; + } + + av_log(avctx, AV_LOG_INFO, "Received a message to change the bitrate from %lu to %lu.\n", bitrate_old, bitrate_new); + __atomic_store_n(&s->ctrl_ctx.bitrate_next, bitrate_new, __ATOMIC_SEQ_CST); + } + + close(hctx->client_sock_fd); + free(hctx); + pthread_exit(NULL); +} + +static void *mediacodecenc_ctrl_handle_listener(void *_avctx) { + AVCodecContext *avctx = _avctx; + MediaCodecEncContext *s = avctx->priv_data; + + if (listen(s->ctrl_ctx.server_sock_fd, 5) < 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to start listening socket '%s': errno:%d: %s\n", s->ctrl_ctx.server_sock_path, errno, strerror(errno)); + goto mediacodecenc_ctrl_handle_listener_end; + } + + av_log(avctx, AV_LOG_TRACE, "Listening socket '%s'\n", s->ctrl_ctx.server_sock_path); + + s->ctrl_ctx.listener_status = mcls_running; + while (s->ctrl_ctx.listener_status == mcls_running) { + pthread_t client_thread_id; + int pthread_error; + mediacodecenc_ctrl_handle_connection_context_t *hctx; + + int client_sock_fd = accept(s->ctrl_ctx.server_sock_fd, NULL, NULL); + if (client_sock_fd < 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to accept a connection to socket '%s': code:%d: description:%s\n", s->ctrl_ctx.server_sock_path, errno, strerror(errno)); + continue; + } + + av_log(avctx, AV_LOG_TRACE, "Accepted a connection to '%s'\n", s->ctrl_ctx.server_sock_path); + + hctx = malloc(sizeof(mediacodecenc_ctrl_handle_connection_context_t)); + hctx->client_sock_fd = client_sock_fd; + hctx->avctx = avctx; + + pthread_error = pthread_create(&client_thread_id, NULL, mediacodecenc_ctrl_handle_connection, hctx); + if (pthread_error != 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to accept a connection to socket '%s': code:%d: description:%s\n", s->ctrl_ctx.server_sock_path, pthread_error, strerror(pthread_error)); + free(hctx); + close(client_sock_fd); + continue; + } + + // mediacodecenc_ctrl_handle_connection takes ownership of hctx and frees it when finished, + // so we are only detaching here and that's it. + pthread_error = pthread_detach(client_thread_id); + if (pthread_error != 0) { + av_log(avctx, AV_LOG_ERROR, "Unable to detach the client handler thread: code:%d: description:%s\n", pthread_error, strerror(pthread_error)); + } + } + +mediacodecenc_ctrl_handle_listener_end: + close(s->ctrl_ctx.server_sock_fd); + unlink(s->ctrl_ctx.server_sock_path); + s->ctrl_ctx.listener_status = mcls_stopped; + pthread_exit(NULL); +} + +int mediacodecenc_ctrl_init(AVCodecContext *avctx, FFAMediaFormat *format) { + MediaCodecEncContext *s = avctx->priv_data; + char *socket_path = s->bitrate_ctrl_socket; + int server_sock_fd; + struct sockaddr_un server_addr; + int pthread_error; + + s->ctrl_ctx.bitrate_cur = avctx->bit_rate; + s->ctrl_ctx.bitrate_next = s->ctrl_ctx.bitrate_cur; + s->ctrl_ctx.out_format = format; + + server_sock_fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (server_sock_fd < 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to create socket '%s': errno:%d: %s\n", socket_path, errno, strerror(errno)); + return errno; + } + + memset(&server_addr, 0, sizeof(struct sockaddr_un)); + server_addr.sun_family = AF_UNIX; + strncpy(server_addr.sun_path, socket_path, sizeof(server_addr.sun_path) - 1); + + if (bind(server_sock_fd, (struct sockaddr *)&server_addr, sizeof(struct sockaddr_un)) < 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to bind to the socket '%s': errno:%d: %s\n", socket_path, errno, strerror(errno)); + return errno; + } + s->ctrl_ctx.server_sock_path = socket_path; + s->ctrl_ctx.server_sock_fd = server_sock_fd; + + pthread_error = pthread_create(&s->ctrl_ctx.listener_thread_id, NULL, mediacodecenc_ctrl_handle_listener, avctx); + if (pthread_error != 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to create a thread to listen the socket '%s': errno:%d: %s\n", socket_path, pthread_error, strerror(pthread_error)); + return pthread_error; + } + + return 0; +} + +int mediacodecenc_ctrl_deinit(AVCodecContext *avctx) { + MediaCodecEncContext *s = avctx->priv_data; + void *retval; + int pthread_error; + if (s->ctrl_ctx.listener_thread_id != 0) { + return 0; + } + + s->ctrl_ctx.listener_status = mcls_stopping; + close(s->ctrl_ctx.server_sock_fd); + + pthread_error = pthread_join(s->ctrl_ctx.listener_thread_id, &retval); + if (pthread_error != 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to detach from the listener thread of the socket '%s': errno:%d: %s\n", s->ctrl_ctx.server_sock_path, pthread_error, strerror(pthread_error)); + return pthread_error; + } + + ff_AMediaFormat_delete(s->ctrl_ctx.out_format); + return 0; +} + +#endif //__ANDROID_API__ >= 26 diff --git a/libavcodec/mediacodecenc_ctrl.h b/libavcodec/mediacodecenc_ctrl.h new file mode 100644 index 0000000000..a5df203639 --- /dev/null +++ b/libavcodec/mediacodecenc_ctrl.h @@ -0,0 +1,50 @@ +/* + * Android MediaCodec encoders + * + * Copyright (c) 2024 Dmitrii Okunev <xaionaro@gmail.com> + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVCODEC_MEDIACODECENC_CTRL_H +#define AVCODEC_MEDIACODECENC_CTRL_H +#if __ANDROID_API__ >= 26 + +#include "avcodec.h" +#include "mediacodec_wrapper.h" + +typedef enum mediacodecenc_ctrl_listener_status { + mcls_stopped = 0, + mcls_running = 1, + mcls_stopping = 2, +} mediacodecenc_ctrl_listener_status_t; + +typedef struct mediacodecenc_ctrl_ctx { + char *server_sock_path; + int server_sock_fd; + pthread_t listener_thread_id; + mediacodecenc_ctrl_listener_status_t listener_status; + uint64_t bitrate_cur; + uint64_t bitrate_next; + FFAMediaFormat *out_format; +} MediaCodecEncControlContext; + +extern int mediacodecenc_ctrl_init(AVCodecContext *avctx, FFAMediaFormat *format); +extern int mediacodecenc_ctrl_deinit(AVCodecContext *avctx); + +#endif //__ANDROID_API__ >= 26 +#endif /* AVCODEC_MEDIACODECENC_CTRL_H */
MediaCodec supports parameter "video-bitrate" to change the bitrate on fly. This commit adds capability to use it. It adds option -bitrate_ctrl_socket to the encoder which makes the encoder to create an UNIX socket and listen for messages to change the bitrate. An example of ffmpeg execution: ffmpeg -f flv -i rtmp://0.0.0.0:1935/live/myStream -c:v hevc_mediacodec -bitrate_ctrl_socket /run/bitrate.sock -b:v 8M -f flv rtmp://127.0.0.1:1935/live/reEncoded An example of changing the bitrate to 1000 Kbps: printf '%016X' 1000000 | xxd -r -p | socat -u STDIN UNIX:/run/bitrate.sock Signed-off-by: Dmitrii Okunev <xaionaro@gmail.com> --- libavcodec/Makefile | 2 +- libavcodec/mediacodec_wrapper.c | 50 ++++++++ libavcodec/mediacodec_wrapper.h | 12 ++ libavcodec/mediacodecenc.c | 128 ++++++++++----------- libavcodec/mediacodecenc.h | 97 ++++++++++++++++ libavcodec/mediacodecenc_ctrl.c | 194 ++++++++++++++++++++++++++++++++ libavcodec/mediacodecenc_ctrl.h | 50 ++++++++ 7 files changed, 469 insertions(+), 64 deletions(-) create mode 100644 libavcodec/mediacodecenc.h create mode 100644 libavcodec/mediacodecenc_ctrl.c create mode 100644 libavcodec/mediacodecenc_ctrl.h