[FFmpeg-devel] avcodec/libaom: Support monochrome encoding with libaom >= 2.0.1

Monochrome encoding with libaom was buggy for a long time, but this was
finally sorted out in libaom 2.0.1 (2.0.0 is almost there but was still
buggy in realtime mode).

Surprisingly, we've never had an external library feature flag in
configure before, but it seems reasonable to add such a category.

Signed-off-by: Philip Langdale <philipl@overt.org>
---
 Changelog              |  1 +
 configure              |  8 ++++++++
 libavcodec/libaomenc.c | 14 ++++++++++++++
 libavcodec/version.h   |  2 +-
 4 files changed, 24 insertions(+), 1 deletion(-)

On 12/8/2020 2:04 AM, Philip Langdale wrote:
> Monochrome encoding with libaom was buggy for a long time, but this was
> finally sorted out in libaom 2.0.1 (2.0.0 is almost there but was still
> buggy in realtime mode).
> 
> Surprisingly, we've never had an external library feature flag in
> configure before, but it seems reasonable to add such a category.
> 
> Signed-off-by: Philip Langdale <philipl@overt.org>
> ---
>   Changelog              |  1 +
>   configure              |  8 ++++++++
>   libavcodec/libaomenc.c | 14 ++++++++++++++
>   libavcodec/version.h   |  2 +-
>   4 files changed, 24 insertions(+), 1 deletion(-)
> 
> diff --git a/Changelog b/Changelog
> index 503317dfae..8f5e849f8d 100644
> --- a/Changelog
> +++ b/Changelog
> @@ -51,6 +51,7 @@ version <next>:
>   - asubcut filter
>   - Microsoft Paint (MSP) version 2 decoder
>   - Microsoft Paint (MSP) demuxer
> +- AV1 monochrome encoding support via libaom >= 2.0.1
>   
>   
>   version 4.3:
> diff --git a/configure b/configure
> index 10dd40cab8..6c83895414 100755
> --- a/configure
> +++ b/configure
> @@ -1829,6 +1829,10 @@ EXTERNAL_LIBRARY_LIST="
>       vapoursynth
>   "
>   
> +EXTERNAL_LIBRARY_FEATURES="
> +    libaom2
> +"
> +
>   HWACCEL_AUTODETECT_LIBRARY_LIST="
>       amf
>       audiotoolbox
> @@ -2314,6 +2318,7 @@ HAVE_LIST="
>       $ARCH_FEATURES
>       $BUILTIN_LIST
>       $COMPLEX_FUNCS
> +    $EXTERNAL_LIBRARY_FEATURES
>       $HAVE_LIST_CMDLINE
>       $HAVE_LIST_PUB
>       $HEADERS_LIST
> @@ -6329,6 +6334,9 @@ enabled gnutls            && require_pkg_config gnutls gnutls gnutls/gnutls.h gn
>   enabled jni               && { [ $target_os = "android" ] && check_headers jni.h && enabled pthreads || die "ERROR: jni not found"; }
>   enabled ladspa            && require_headers "ladspa.h dlfcn.h"
>   enabled libaom            && require_pkg_config libaom "aom >= 1.0.0" aom/aom_codec.h aom_codec_version
> +if enabled libaom; then
> +    check_pkg_config libaom2 "aom >= 2.0.1" aom/aom_codec.h aom_codec_version

It should be cleaner to do the version check in libaomenc.c

Look at av1_init_static(), where we check for libaom 2 in order to tag 
the wrapper as experimental if the library is too old. In there you can 
add a check for 2.0.1 (like aom_codec_version() >= 0x20001) and make it 
set codec->pix_fmts to a new list of formats that include the gray ones 
depending on the result.

> +fi
>   enabled libaribb24        && { check_pkg_config libaribb24 "aribb24 > 1.0.3" "aribb24/aribb24.h" arib_instance_new ||
>                                  { enabled gpl && require_pkg_config libaribb24 aribb24 "aribb24/aribb24.h" arib_instance_new; } ||
>                                  die "ERROR: libaribb24 requires version higher than 1.0.3 or --enable-gpl."; }
> diff --git a/libavcodec/libaomenc.c b/libavcodec/libaomenc.c
> index 2b0581b15a..6110472b68 100644
> --- a/libavcodec/libaomenc.c
> +++ b/libavcodec/libaomenc.c
> @@ -338,6 +338,10 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
>       const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt);
>       enccfg->g_bit_depth = enccfg->g_input_bit_depth = desc->comp[0].depth;
>       switch (avctx->pix_fmt) {
> +#if HAVE_LIBAOM2
> +    case AV_PIX_FMT_GRAY8:
> +        enccfg->monochrome = 1;
> +#endif

Add a line like

// Fall-through

Here, to hint the compiler to not warn about a missing break.

>       case AV_PIX_FMT_YUV420P:
>           enccfg->g_profile = FF_PROFILE_AV1_MAIN;
>           *img_fmt = AOM_IMG_FMT_I420;
> @@ -351,6 +355,11 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
>           enccfg->g_profile = FF_PROFILE_AV1_HIGH;
>           *img_fmt = AOM_IMG_FMT_I444;
>           return 0;
> +#if HAVE_LIBAOM2
> +    case AV_PIX_FMT_GRAY10:
> +    case AV_PIX_FMT_GRAY12:
> +        enccfg->monochrome = 1;
> +#endif

Ditto.

>       case AV_PIX_FMT_YUV420P10:
>       case AV_PIX_FMT_YUV420P12:
>           if (codec_caps & AOM_CODEC_CAP_HIGHBITDEPTH) {
> @@ -1171,6 +1180,11 @@ static const enum AVPixelFormat av1_pix_fmts_highbd[] = {
>       AV_PIX_FMT_YUV444P12,
>       AV_PIX_FMT_GBRP10,
>       AV_PIX_FMT_GBRP12,
> +#if HAVE_LIBAOM2
> +    AV_PIX_FMT_GRAY8,
> +    AV_PIX_FMT_GRAY10,
> +    AV_PIX_FMT_GRAY12,
> +#endif
>       AV_PIX_FMT_NONE
>   };
>   
> diff --git a/libavcodec/version.h b/libavcodec/version.h
> index 1c10d105f6..5b92afe60a 100644
> --- a/libavcodec/version.h
> +++ b/libavcodec/version.h
> @@ -29,7 +29,7 @@
>   
>   #define LIBAVCODEC_VERSION_MAJOR  58
>   #define LIBAVCODEC_VERSION_MINOR 115
> -#define LIBAVCODEC_VERSION_MICRO 101
> +#define LIBAVCODEC_VERSION_MICRO 102
>   
>   #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
>                                                  LIBAVCODEC_VERSION_MINOR, \
>