[FFmpeg-devel] lavc/videotoolboxenc: Add support for HEVC with Alpha.

From: Hironori Bono <bouno@rouge.plala.or.jp>

This change supports the "HEVC Video with Alpha" profile introduced in WWDC 2019
<https://developer.apple.com/videos/play/wwdc2019/506/>. (This change is a
partial fix for Ticket #7965.)

For example, the following command converts an animation PNG file to an HEVC
with Alpha video:
./ffmpeg -i fate-suite/apng/clock.png -c:v hevc_videotoolbox -allow_sw 1 -alpha_quality 0.75 -vtag hvc1 clock.mov

(This change uses the "HEVC Video with Alpha" profile only when the
'-alpha_quality' value is not 0 for backward compatibility.)

Signed-off-by: Hironori Bono <bouno@rouge.plala.or.jp>
---
 configure                    |  2 ++
 libavcodec/videotoolboxenc.c | 47 +++++++++++++++++++++++++++++++++---
 2 files changed, 46 insertions(+), 3 deletions(-)

Greetings,

Thank you for your help in advance.
Unfortunately, my patch, which add support the "HEVC with alpha" profile to the videotoolbox encoder, has not been reviewed for nearly three weeks. Would it be possible to give me why nobody has reviewed my patch?

Regards,

Hironori Bono
E-mail: bouno@rouge.plala.or.jp

-----Original Message-----
From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of bouno@rouge.plala.or.jp
Sent: Tuesday, February 23, 2021 2:02 PM
To: ffmpeg-devel@ffmpeg.org
Cc: Hironori Bono <bouno@rouge.plala.or.jp>
Subject: [FFmpeg-devel] [PATCH] lavc/videotoolboxenc: Add support for HEVC with Alpha.

From: Hironori Bono <bouno@rouge.plala.or.jp>

This change supports the "HEVC Video with Alpha" profile introduced in WWDC 2019 <https://developer.apple.com/videos/play/wwdc2019/506/>. (This change is a partial fix for Ticket #7965.)

For example, the following command converts an animation PNG file to an HEVC with Alpha video:
./ffmpeg -i fate-suite/apng/clock.png -c:v hevc_videotoolbox -allow_sw 1 -alpha_quality 0.75 -vtag hvc1 clock.mov

(This change uses the "HEVC Video with Alpha" profile only when the '-alpha_quality' value is not 0 for backward compatibility.)

Signed-off-by: Hironori Bono <bouno@rouge.plala.or.jp>
---
 configure                    |  2 ++
 libavcodec/videotoolboxenc.c | 47 +++++++++++++++++++++++++++++++++---
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/configure b/configure
index 336301cb40..63adf131b9 100755
--- a/configure
+++ b/configure
@@ -2288,6 +2288,7 @@ TOOLCHAIN_FEATURES="
 
 TYPES_LIST="
     kCMVideoCodecType_HEVC
+    kCMVideoCodecType_HEVCWithAlpha
     kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
     kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ
     kCVImageBufferTransferFunction_ITU_R_2100_HLG
@@ -6211,6 +6212,7 @@ enabled avfoundation && {  enabled videotoolbox && {
     check_lib coreservices CoreServices/CoreServices.h UTGetOSTypeFromString "-framework CoreServices"
     check_func_headers CoreMedia/CMFormatDescription.h kCMVideoCodecType_HEVC "-framework CoreMedia"
+    check_func_headers CoreMedia/CMFormatDescription.h kCMVideoCodecType_HEVCWithAlpha "-framework CoreMedia"
     check_func_headers CoreVideo/CVPixelBuffer.h kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange "-framework CoreVideo"
     check_func_headers CoreVideo/CVImageBuffer.h kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ "-framework CoreVideo"
     check_func_headers CoreVideo/CVImageBuffer.h kCVImageBufferTransferFunction_ITU_R_2100_HLG "-framework CoreVideo"
diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c index c487d2dc60..9d3c7e29dc 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -40,6 +40,10 @@
 enum { kCMVideoCodecType_HEVC = 'hvc1' };  #endif
 
+#if !HAVE_KCMVIDEOCODECTYPE_HEVCWITHALPHA
+enum { kCMVideoCodecType_HEVCWithAlpha = 'muxa' }; #endif
+
 #if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
 enum { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange = 'xf20' };  enum { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange = 'x420' }; @@ -88,6 +92,7 @@ static struct{
     CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel;
 
     CFStringRef kVTCompressionPropertyKey_RealTime;
+    CFStringRef kVTCompressionPropertyKey_TargetQualityForAlpha;
 
     CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder;
     CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder;
@@ -147,6 +152,8 @@ static void loadVTEncSymbols(){
     GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel,   "HEVC_Main10_AutoLevel");
 
     GET_SYM(kVTCompressionPropertyKey_RealTime, "RealTime");
+    GET_SYM(kVTCompressionPropertyKey_TargetQualityForAlpha,
+            "TargetQualityForAlpha");
 
     GET_SYM(kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
             "EnableHardwareAcceleratedVideoEncoder");
@@ -222,6 +229,7 @@ typedef struct VTEncContext {
 
     int64_t allow_sw;
     int64_t require_sw;
+    double alpha_quality;
 
     bool flushing;
     bool has_b_frames;
@@ -392,11 +400,17 @@ static int count_nalus(size_t length_code_size,
     return 0;
 }
 
-static CMVideoCodecType get_cm_codec_type(enum AVCodecID id)
+static CMVideoCodecType get_cm_codec_type(enum AVCodecID id,
+                                          enum AVPixelFormat fmt,
+                                          double alpha_quality)
 {
     switch (id) {
     case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
-    case AV_CODEC_ID_HEVC: return kCMVideoCodecType_HEVC;
+    case AV_CODEC_ID_HEVC:
+        if (fmt == AV_PIX_FMT_BGRA && alpha_quality > 0.0) {
+            return kCMVideoCodecType_HEVCWithAlpha;
+        }
+        return kCMVideoCodecType_HEVC;
     default:               return 0;
     }
 }
@@ -786,6 +800,8 @@ static int get_cv_pixel_format(AVCodecContext* avctx,
         *av_pixel_format = range == AVCOL_RANGE_JPEG ?
                                         kCVPixelFormatType_420YpCbCr8PlanarFullRange :
                                         kCVPixelFormatType_420YpCbCr8Planar;
+    } else if (fmt == AV_PIX_FMT_BGRA) {
+        *av_pixel_format = kCVPixelFormatType_32BGRA;
     } else if (fmt == AV_PIX_FMT_P010LE) {
         *av_pixel_format = range == AVCOL_RANGE_JPEG ?
                                         kCVPixelFormatType_420YpCbCr10BiPlanarFullRange :
@@ -1114,6 +1130,20 @@ static int vtenc_create_encoder(AVCodecContext   *avctx,
         }
     }
 
+    if (vtctx->codec_id == AV_CODEC_ID_HEVC) {
+        if (avctx->pix_fmt == AV_PIX_FMT_BGRA && vtctx->alpha_quality > 0.0) {
+            CFNumberRef alpha_quality_num = CFNumberCreate(kCFAllocatorDefault,
+                                                           kCFNumberDoubleType,
+                                                           &vtctx->alpha_quality);
+            if (!alpha_quality_num) return AVERROR(ENOMEM);
+
+            status = VTSessionSetProperty(vtctx->session,
+                                          compat_keys.kVTCompressionPropertyKey_TargetQualityForAlpha,
+                                          alpha_quality_num);
+            CFRelease(alpha_quality_num);
+        }
+    }
+
     if (profile_level) {
         status = VTSessionSetProperty(vtctx->session,
                                       kVTCompressionPropertyKey_ProfileLevel,
@@ -1326,7 +1356,7 @@ static int vtenc_configure_encoder(AVCodecContext *avctx)
     CFNumberRef            gamma_level = NULL;
     int                    status;
 
-    codec_type = get_cm_codec_type(avctx->codec_id);
+    codec_type = get_cm_codec_type(avctx->codec_id, avctx->pix_fmt, 
+ vtctx->alpha_quality);
     if (!codec_type) {
         av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
         return AVERROR(EINVAL);
@@ -2036,6 +2066,14 @@ static int get_cv_pixel_info(
         strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
         break;
 
+    case AV_PIX_FMT_BGRA:
+        *plane_count = 1;
+
+        widths [0] = avctx->width;
+        heights[0] = avctx->height;
+        strides[0] = frame ? frame->linesize[0] : avctx->width * 4;
+        break;
+
     case AV_PIX_FMT_P010LE:
         *plane_count = 2;
         widths[0] = avctx->width;
@@ -2534,6 +2572,7 @@ static const enum AVPixelFormat hevc_pix_fmts[] = {
     AV_PIX_FMT_VIDEOTOOLBOX,
     AV_PIX_FMT_NV12,
     AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_BGRA,
     AV_PIX_FMT_P010LE,
     AV_PIX_FMT_NONE
 };
@@ -2611,6 +2650,8 @@ static const AVOption hevc_options[] = {
     { "main",     "Main Profile",     0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN   }, INT_MIN, INT_MAX, VE, "profile" },
     { "main10",   "Main10 Profile",   0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
 
+    { "alpha_quality", "Compression quality for the alpha channel", 
+ OFFSET(alpha_quality), AV_OPT_TYPE_DOUBLE, { .dbl = 0.0 }, 0.0, 1.0, 
+ VE },
+
     COMMON_OPTIONS
     { NULL },
 };
--
2.24.3 (Apple Git-128)

On Tue, Mar 16, 2021 at 5:59 AM <bouno@rouge.plala.or.jp> wrote:

> Greetings,
>
> Thank you for your help in advance.
> Unfortunately, my patch, which add support the "HEVC with alpha" profile
> to the videotoolbox encoder, has not been reviewed for nearly three weeks.
> Would it be possible to give me why nobody has reviewed my patch?
>
> Regards,
>
> Hironori Bono
> E-mail: bouno@rouge.plala.or.jp
>
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> bouno@rouge.plala.or.jp
> Sent: Tuesday, February 23, 2021 2:02 PM
> To: ffmpeg-devel@ffmpeg.org
> Cc: Hironori Bono <bouno@rouge.plala.or.jp>
> Subject: [FFmpeg-devel] [PATCH] lavc/videotoolboxenc: Add support for HEVC
> with Alpha.
>
> From: Hironori Bono <bouno@rouge.plala.or.jp>
>
> This change supports the "HEVC Video with Alpha" profile introduced in
> WWDC 2019 <https://developer.apple.com/videos/play/wwdc2019/506/>. (This
> change is a partial fix for Ticket #7965.)
>
> For example, the following command converts an animation PNG file to an
> HEVC with Alpha video:
> ./ffmpeg -i fate-suite/apng/clock.png -c:v hevc_videotoolbox -allow_sw 1
> -alpha_quality 0.75 -vtag hvc1 clock.mov
>
> (This change uses the "HEVC Video with Alpha" profile only when the
> '-alpha_quality' value is not 0 for backward compatibility.)
>
> Signed-off-by: Hironori Bono <bouno@rouge.plala.or.jp>
>

Applied

> ---
>  configure                    |  2 ++
>  libavcodec/videotoolboxenc.c | 47 +++++++++++++++++++++++++++++++++---
>  2 files changed, 46 insertions(+), 3 deletions(-)
>
> diff --git a/configure b/configure
> index 336301cb40..63adf131b9 100755
> --- a/configure
> +++ b/configure
> @@ -2288,6 +2288,7 @@ TOOLCHAIN_FEATURES="
>
>  TYPES_LIST="
>      kCMVideoCodecType_HEVC
> +    kCMVideoCodecType_HEVCWithAlpha
>      kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
>      kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ
>      kCVImageBufferTransferFunction_ITU_R_2100_HLG
> @@ -6211,6 +6212,7 @@ enabled avfoundation && {  enabled videotoolbox && {
>      check_lib coreservices CoreServices/CoreServices.h
> UTGetOSTypeFromString "-framework CoreServices"
>      check_func_headers CoreMedia/CMFormatDescription.h
> kCMVideoCodecType_HEVC "-framework CoreMedia"
> +    check_func_headers CoreMedia/CMFormatDescription.h
> kCMVideoCodecType_HEVCWithAlpha "-framework CoreMedia"
>      check_func_headers CoreVideo/CVPixelBuffer.h
> kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange "-framework CoreVideo"
>      check_func_headers CoreVideo/CVImageBuffer.h
> kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ "-framework CoreVideo"
>      check_func_headers CoreVideo/CVImageBuffer.h
> kCVImageBufferTransferFunction_ITU_R_2100_HLG "-framework CoreVideo"
> diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
> index c487d2dc60..9d3c7e29dc 100644
> --- a/libavcodec/videotoolboxenc.c
> +++ b/libavcodec/videotoolboxenc.c
> @@ -40,6 +40,10 @@
>  enum { kCMVideoCodecType_HEVC = 'hvc1' };  #endif
>
> +#if !HAVE_KCMVIDEOCODECTYPE_HEVCWITHALPHA
> +enum { kCMVideoCodecType_HEVCWithAlpha = 'muxa' }; #endif
> +
>  #if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
>  enum { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange = 'xf20' };  enum
> { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange = 'x420' }; @@ -88,6
> +92,7 @@ static struct{
>      CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel;
>
>      CFStringRef kVTCompressionPropertyKey_RealTime;
> +    CFStringRef kVTCompressionPropertyKey_TargetQualityForAlpha;
>
>      CFStringRef
> kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder;
>      CFStringRef
> kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder;
> @@ -147,6 +152,8 @@ static void loadVTEncSymbols(){
>      GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel,
>  "HEVC_Main10_AutoLevel");
>
>      GET_SYM(kVTCompressionPropertyKey_RealTime, "RealTime");
> +    GET_SYM(kVTCompressionPropertyKey_TargetQualityForAlpha,
> +            "TargetQualityForAlpha");
>
>
>  GET_SYM(kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
>              "EnableHardwareAcceleratedVideoEncoder");
> @@ -222,6 +229,7 @@ typedef struct VTEncContext {
>
>      int64_t allow_sw;
>      int64_t require_sw;
> +    double alpha_quality;
>
>      bool flushing;
>      bool has_b_frames;
> @@ -392,11 +400,17 @@ static int count_nalus(size_t length_code_size,
>      return 0;
>  }
>
> -static CMVideoCodecType get_cm_codec_type(enum AVCodecID id)
> +static CMVideoCodecType get_cm_codec_type(enum AVCodecID id,
> +                                          enum AVPixelFormat fmt,
> +                                          double alpha_quality)
>  {
>      switch (id) {
>      case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
> -    case AV_CODEC_ID_HEVC: return kCMVideoCodecType_HEVC;
> +    case AV_CODEC_ID_HEVC:
> +        if (fmt == AV_PIX_FMT_BGRA && alpha_quality > 0.0) {
> +            return kCMVideoCodecType_HEVCWithAlpha;
> +        }
> +        return kCMVideoCodecType_HEVC;
>      default:               return 0;
>      }
>  }
> @@ -786,6 +800,8 @@ static int get_cv_pixel_format(AVCodecContext* avctx,
>          *av_pixel_format = range == AVCOL_RANGE_JPEG ?
>
>  kCVPixelFormatType_420YpCbCr8PlanarFullRange :
>
>  kCVPixelFormatType_420YpCbCr8Planar;
> +    } else if (fmt == AV_PIX_FMT_BGRA) {
> +        *av_pixel_format = kCVPixelFormatType_32BGRA;
>      } else if (fmt == AV_PIX_FMT_P010LE) {
>          *av_pixel_format = range == AVCOL_RANGE_JPEG ?
>
>  kCVPixelFormatType_420YpCbCr10BiPlanarFullRange :
> @@ -1114,6 +1130,20 @@ static int vtenc_create_encoder(AVCodecContext
>  *avctx,
>          }
>      }
>
> +    if (vtctx->codec_id == AV_CODEC_ID_HEVC) {
> +        if (avctx->pix_fmt == AV_PIX_FMT_BGRA && vtctx->alpha_quality >
> 0.0) {
> +            CFNumberRef alpha_quality_num =
> CFNumberCreate(kCFAllocatorDefault,
> +
>  kCFNumberDoubleType,
> +
>  &vtctx->alpha_quality);
> +            if (!alpha_quality_num) return AVERROR(ENOMEM);
> +
> +            status = VTSessionSetProperty(vtctx->session,
> +
> compat_keys.kVTCompressionPropertyKey_TargetQualityForAlpha,
> +                                          alpha_quality_num);
> +            CFRelease(alpha_quality_num);
> +        }
> +    }
> +
>      if (profile_level) {
>          status = VTSessionSetProperty(vtctx->session,
>
>  kVTCompressionPropertyKey_ProfileLevel,
> @@ -1326,7 +1356,7 @@ static int vtenc_configure_encoder(AVCodecContext
> *avctx)
>      CFNumberRef            gamma_level = NULL;
>      int                    status;
>
> -    codec_type = get_cm_codec_type(avctx->codec_id);
> +    codec_type = get_cm_codec_type(avctx->codec_id, avctx->pix_fmt,
> + vtctx->alpha_quality);
>      if (!codec_type) {
>          av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID
> %d\n", avctx->codec_id);
>          return AVERROR(EINVAL);
> @@ -2036,6 +2066,14 @@ static int get_cv_pixel_info(
>          strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
>          break;
>
> +    case AV_PIX_FMT_BGRA:
> +        *plane_count = 1;
> +
> +        widths [0] = avctx->width;
> +        heights[0] = avctx->height;
> +        strides[0] = frame ? frame->linesize[0] : avctx->width * 4;
> +        break;
> +
>      case AV_PIX_FMT_P010LE:
>          *plane_count = 2;
>          widths[0] = avctx->width;
> @@ -2534,6 +2572,7 @@ static const enum AVPixelFormat hevc_pix_fmts[] = {
>      AV_PIX_FMT_VIDEOTOOLBOX,
>      AV_PIX_FMT_NV12,
>      AV_PIX_FMT_YUV420P,
> +    AV_PIX_FMT_BGRA,
>      AV_PIX_FMT_P010LE,
>      AV_PIX_FMT_NONE
>  };
> @@ -2611,6 +2650,8 @@ static const AVOption hevc_options[] = {
>      { "main",     "Main Profile",     0, AV_OPT_TYPE_CONST, { .i64 =
> HEVC_PROF_MAIN   }, INT_MIN, INT_MAX, VE, "profile" },
>      { "main10",   "Main10 Profile",   0, AV_OPT_TYPE_CONST, { .i64 =
> HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
>
> +    { "alpha_quality", "Compression quality for the alpha channel",
> + OFFSET(alpha_quality), AV_OPT_TYPE_DOUBLE, { .dbl = 0.0 }, 0.0, 1.0,
> + VE },
> +
>      COMMON_OPTIONS
>      { NULL },
>  };
> --
> 2.24.3 (Apple Git-128)
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email ffmpeg-devel-request@ffmpeg.org
> with subject "unsubscribe".
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".