[FFmpeg-devel] libavcodec/qsvenc: enable Hyper Encode

Hyper Encode uses Intel integrated and discrete graphics on one system
to accelerate encoding of a single video stream.
Depending on the selected parameters and codecs, performance gain on AlderLake iGPU + ARC Gfx up to 1.6x.

More information: https://www.intel.co.uk/content/www/uk/en/architecture-and-technology/adaptix/deep-link.html
Developer guide: https://github.com/oneapi-src/oneVPL-intel-gpu/blob/main/doc/HyperEncode_FeatureDeveloperGuide.md

Hyper Encode is supported only on Windows and requires D3D11 and oneVPL.

To enable Hyper Encode need to specify:
-Hyper Encode mode (-dual_gfx on or dual_gfx adaptive)
-Encoder: h264_qsv or hevc_qsv
-BRC: VBR, CQP or ICQ
-Lowpower mode (-low_power 1)
-Closed GOP for AVC or strict GOP for HEVC -idr_interval = 0 used by default

Depending on the encoding parameters, the following parameters may need
to be adjusted:
-g recommended >= 30 for better performance
-async_depth recommended >= 30 for better performance
-extra_hw_frames recommended equal to async_depth value
-bf recommended = 0 for better performance

In the cases with fast encoding (-preset veryfast) there may be no
performance gain due to the fact that the decode is slower than the encode.

Command line examples:

ffmpeg.exe -init_hw_device qsv:hw,child_device_type=d3d11va,child_device=0 -v verbose -y -hwaccel qsv -extra_hw_frames 60 -async_depth 60 -c:v h264_qsv -i bbb_sunflower_2160p_60fps_normal.mp4
-async_depth 60 -c:v h264_qsv -preset medium -g 60 -low_power 1 -bf 0 -dual_gfx on output.h265

Signed-off-by: galinart <artem.galin@intel.com>
---
 libavcodec/qsvenc.c      | 91 +++++++++++++++++++++++++++++++++++++++-
 libavcodec/qsvenc.h      | 19 ++++++++-
 libavcodec/qsvenc_h264.c |  3 ++
 libavcodec/qsvenc_hevc.c |  3 ++
 4 files changed, 112 insertions(+), 4 deletions(-)

On Ma, 2022-12-05 at 20:18 +0000, galinart wrote:
> Hyper Encode uses Intel integrated and discrete graphics on one system
> to accelerate encoding of a single video stream.
> Depending on the selected parameters and codecs, performance gain on AlderLake
> iGPU + ARC Gfx up to 1.6x.
> 
> More information: 
> https://www.intel.co.uk/content/www/uk/en/architecture-and-technology/adaptix/deep-link.html
> Developer guide: 
> https://github.com/oneapi-src/oneVPL-intel-gpu/blob/main/doc/HyperEncode_FeatureDeveloperGuide.md
> 
> Hyper Encode is supported only on Windows and requires D3D11 and oneVPL.
> 
> To enable Hyper Encode need to specify:
> -Hyper Encode mode (-dual_gfx on or dual_gfx adaptive)
> -Encoder: h264_qsv or hevc_qsv
> -BRC: VBR, CQP or ICQ
> -Lowpower mode (-low_power 1)
> -Closed GOP for AVC or strict GOP for HEVC -idr_interval = 0 used by default
> 
> Depending on the encoding parameters, the following parameters may need
> to be adjusted:
> -g recommended >= 30 for better performance
> -async_depth recommended >= 30 for better performance
> -extra_hw_frames recommended equal to async_depth value
> -bf recommended = 0 for better performance
> 
> In the cases with fast encoding (-preset veryfast) there may be no
> performance gain due to the fact that the decode is slower than the encode.
> 
> Command line examples:
> 
> ffmpeg.exe -init_hw_device qsv:hw,child_device_type=d3d11va,child_device=0 -v
> verbose -y -hwaccel qsv -extra_hw_frames 60 -async_depth 60 -c:v h264_qsv -i
> bbb_sunflower_2160p_60fps_normal.mp4
> -async_depth 60 -c:v h264_qsv -preset medium -g 60 -low_power 1 -bf 0
> -dual_gfx on output.h265
> 
> Signed-off-by: galinart <artem.galin@intel.com>
> ---
>  libavcodec/qsvenc.c      | 91 +++++++++++++++++++++++++++++++++++++++-
>  libavcodec/qsvenc.h      | 19 ++++++++-
>  libavcodec/qsvenc_h264.c |  3 ++
>  libavcodec/qsvenc_hevc.c |  3 ++
>  4 files changed, 112 insertions(+), 4 deletions(-)
> 
> diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
> index d5e9f2d420..96526d4579 100644
> --- a/libavcodec/qsvenc.c
> +++ b/libavcodec/qsvenc.c
> @@ -169,6 +169,8 @@ do {                        \
>      }                       \
>  } while (0)                 \
>  
> +#define MFX_IMPL_VIA_MASK(impl) (0x0f00 & (impl))
> +
>  static const char *print_ratecontrol(mfxU16 rc_mode)
>  {
>      int i;
> @@ -197,6 +199,10 @@ static void dump_video_param(AVCodecContext *avctx,
> QSVEncContext *q,
>      mfxExtCodingOption2 *co2 = NULL;
>      mfxExtCodingOption3 *co3 = NULL;
>      mfxExtHEVCTiles *exthevctiles = NULL;
> +#if QSV_HAVE_HE
> +    mfxExtHyperModeParam *exthypermodeparam = NULL;
> +#endif
> +
>      const char *tmp_str = NULL;
>  
>      if (q->co2_idx > 0)
> @@ -208,6 +214,11 @@ static void dump_video_param(AVCodecContext *avctx,
> QSVEncContext *q,
>      if (q->exthevctiles_idx > 0)
>          exthevctiles = (mfxExtHEVCTiles *)coding_opts[q->exthevctiles_idx];
>  
> +#if QSV_HAVE_HE
> +    if (q->exthypermodeparam_idx > 0)
> +        exthypermodeparam = (mfxExtHyperModeParam *)coding_opts[q-
> >exthypermodeparam_idx];
> +#endif
> +
>      av_log(avctx, AV_LOG_VERBOSE, "profile: %s; level: %"PRIu16"\n",
>             print_profile(avctx->codec_id, info->CodecProfile), info-
> >CodecLevel);
>  
> @@ -373,6 +384,21 @@ static void dump_video_param(AVCodecContext *avctx,
> QSVEncContext *q,
>          av_log(avctx, AV_LOG_VERBOSE, "NumTileColumns: %"PRIu16";
> NumTileRows: %"PRIu16"\n",
>                 exthevctiles->NumTileColumns, exthevctiles->NumTileRows);
>      }
> +
> +#if QSV_HAVE_HE
> +    if (exthypermodeparam) {
> +        av_log(avctx, AV_LOG_VERBOSE, "HyperEncode: ");
> +
> +        if (exthypermodeparam->Mode == MFX_HYPERMODE_OFF)
> +            av_log(avctx, AV_LOG_VERBOSE, "OFF");
> +        if (exthypermodeparam->Mode == MFX_HYPERMODE_ON)
> +            av_log(avctx, AV_LOG_VERBOSE, "ON");
> +        if (exthypermodeparam->Mode == MFX_HYPERMODE_ADAPTIVE)
> +            av_log(avctx, AV_LOG_VERBOSE, "Adaptive");
> +
> +        av_log(avctx, AV_LOG_VERBOSE, "\n");
> +    }
> +#endif
>  }
>  
>  static void dump_video_vp9_param(AVCodecContext *avctx, QSVEncContext *q,
> @@ -1154,6 +1180,54 @@ static int init_video_param(AVCodecContext *avctx,
> QSVEncContext *q)
>          q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q-
> >extvsi;
>      }
>  
> +#if QSV_HAVE_HE
> +   if (q->dual_gfx) {
> +        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 4)) {
> +            mfxIMPL impl;
> +            MFXQueryIMPL(q->session, &impl);
> +
> +            if (MFX_IMPL_VIA_MASK(impl) != MFX_IMPL_VIA_D3D11) {
> +                av_log(avctx, AV_LOG_ERROR, "Dual GFX mode requires D3D11VA
> \n");
> +                return AVERROR_UNKNOWN;
> +            }
> +            if (q->param.mfx.LowPower != MFX_CODINGOPTION_ON) {
> +                av_log(avctx, AV_LOG_ERROR, "Dual GFX mode supports only low-
> power encoding mode \n");
> +                return AVERROR_UNKNOWN;
> +            }
> +            if (q->param.mfx.CodecId != MFX_CODEC_AVC && q->param.mfx.CodecId 
> != MFX_CODEC_HEVC) {
> +                av_log(avctx, AV_LOG_ERROR, "Not supported encoder for dual
> GFX mode. "
> +                                            "Supported: h264_qsv and hevc_qsv
> \n");
> +                return AVERROR_UNKNOWN;
> +            }
> +            if (q->param.mfx.RateControlMethod != MFX_RATECONTROL_VBR &&
> +                q->param.mfx.RateControlMethod != MFX_RATECONTROL_CQP &&
> +                q->param.mfx.RateControlMethod != MFX_RATECONTROL_ICQ) {
> +                av_log(avctx, AV_LOG_WARNING, "Not supported BRC for dual GFX
> mode. "
> +                                            "Supported: VBR, CQP and ICQ
> \n");
> +            }
> +            if ((q->param.mfx.CodecId == MFX_CODEC_AVC  && q-
> >param.mfx.IdrInterval != 0) ||
> +                (q->param.mfx.CodecId == MFX_CODEC_HEVC && q-
> >param.mfx.IdrInterval != 1)) {
> +                av_log(avctx, AV_LOG_WARNING, "Dual GFX mode requires closed
> GOP for AVC and strict GOP for HEVC, -idr_interval 0 \n");
> +            }
> +            if (q->param.mfx.GopPicSize < 30) {
> +                av_log(avctx, AV_LOG_WARNING, "For better performance in dual
> GFX mode GopPicSize must be >= 30 \n");
> +            }
> +            if (q->param.AsyncDepth < 30) {
> +                av_log(avctx, AV_LOG_WARNING, "For better performance in dual
> GFX mode AsyncDepth must be >= 30 \n");
> +            }
> +
> +            q->exthypermodeparam.Header.BufferId =
> MFX_EXTBUFF_HYPER_MODE_PARAM;
> +            q->exthypermodeparam.Header.BufferSz = sizeof(q-
> >exthypermodeparam);
> +            q->exthypermodeparam.Mode            = q->dual_gfx;
> +            q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer
> *)&q->exthypermodeparam;
> +        } else {
> +            av_log(avctx, AV_LOG_ERROR,
> +                   "This version of runtime doesn't support Hyper Encode\n");
> +            return AVERROR_UNKNOWN;
> +        }
> +    }
> +#endif
> +
>      if (!check_enc_param(avctx,q)) {
>          av_log(avctx, AV_LOG_ERROR,
>                 "some encoding parameters are not supported by the QSV "
> @@ -1328,12 +1402,19 @@ static int qsv_retrieve_enc_params(AVCodecContext
> *avctx, QSVEncContext *q)
>           .Header.BufferSz = sizeof(hevc_tile_buf),
>      };
>  
> -    mfxExtBuffer *ext_buffers[6];
> +#if QSV_HAVE_HE
> +    mfxExtHyperModeParam hyper_mode_param_buf = {
> +        .Header.BufferId = MFX_EXTBUFF_HYPER_MODE_PARAM,
> +        .Header.BufferSz = sizeof(hyper_mode_param_buf),
> +    };
> +#endif
> +
> +    mfxExtBuffer *ext_buffers[6 + QSV_HAVE_HE];
>  
>      int need_pps = avctx->codec_id != AV_CODEC_ID_MPEG2VIDEO;
>      int ret, ext_buf_num = 0, extradata_offset = 0;
>  
> -    q->co2_idx = q->co3_idx = q->exthevctiles_idx = -1;
> +    q->co2_idx = q->co3_idx = q->exthevctiles_idx = q->exthypermodeparam_idx
> = -1;
>      ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&extradata;
>      ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&co;
>  
> @@ -1355,6 +1436,12 @@ static int qsv_retrieve_enc_params(AVCodecContext
> *avctx, QSVEncContext *q)
>          q->exthevctiles_idx = ext_buf_num;
>          ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&hevc_tile_buf;
>      }
> +#if QSV_HAVE_HE
> +    if (q->dual_gfx && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 4)) {
> +        q->exthypermodeparam_idx = ext_buf_num;
> +        ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&hyper_mode_param_buf;
> +    }
> +#endif
>  
>      q->param.ExtParam    = ext_buffers;
>      q->param.NumExtParam = ext_buf_num;
> diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h
> index a7bbb3797e..4a6fa2caed 100644
> --- a/libavcodec/qsvenc.h
> +++ b/libavcodec/qsvenc.h
> @@ -45,10 +45,12 @@
>  #define QSV_HAVE_AVBR   1
>  #define QSV_HAVE_VCM    1
>  #define QSV_HAVE_MF     0
> +#define QSV_HAVE_HE     QSV_VERSION_ATLEAST(2, 4)
>  #else
>  #define QSV_HAVE_AVBR   0
>  #define QSV_HAVE_VCM    0
>  #define QSV_HAVE_MF     !QSV_ONEVPL
> +#define QSV_HAVE_HE     0
>  #endif
>  
>  #define QSV_COMMON_OPTS \
> @@ -64,6 +66,14 @@
>  { "forced_idr",     "Forcing I frames as IDR
> frames",         OFFSET(qsv.forced_idr),     AV_OPT_TYPE_BOOL,{ .i64 =
> 0  },  0,          1, VE },                         \
>  { "low_power", "enable low power mode(experimental: many limitations by mfx
> version, BRC modes, etc.)", OFFSET(qsv.low_power), AV_OPT_TYPE_BOOL, { .i64 =
> -1}, -1, 1, VE},
>  
> +#if QSV_HAVE_HE
> +#define QSV_HE_OPTIONS \
> +{ "dual_gfx", "Prefer processing on both iGfx and dGfx
> simultaneously",                                             OFFSET(qsv.dual_g
> fx), AV_OPT_TYPE_INT, { .i64 = MFX_HYPERMODE_OFF }, MFX_HYPERMODE_OFF,
> MFX_HYPERMODE_ADAPTIVE, VE, "dual_gfx" }, \
> +{ "off",      "Disable HyperEncode
> mode",                                                                        
>    0, AV_OPT_TYPE_CONST, { .i64 = MFX_HYPERMODE_OFF       },   INT_MIN,
> INT_MAX, VE, "dual_gfx" }, \
> +{ "on",       "Enable HyperEncode mode and return error if incompatible
> parameters during initialization",          0, AV_OPT_TYPE_CONST, { .i64 =
> MFX_HYPERMODE_ON        },   INT_MIN, INT_MAX, VE, "dual_gfx" }, \
> +{ "adaptive", "Enable HyperEncode mode or fallback to single GPU if
> incompatible parameters during initialization", 0, AV_OPT_TYPE_CONST, { .i64 =
> MFX_HYPERMODE_ADAPTIVE  },   INT_MIN, INT_MAX, VE, "dual_gfx" },
> +#endif
> +
>  #define QSV_OPTION_RDO \
>  { "rdo",            "Enable rate distortion
> optimization",    OFFSET(qsv.rdo),            AV_OPT_TYPE_INT, { .i64 = -1 },
> -1,          1, VE },
>  
> @@ -171,7 +181,9 @@ typedef struct QSVEncContext {
>      mfxExtAV1TileParam extav1tileparam;
>      mfxExtAV1BitstreamParam extav1bsparam;
>  #endif
> -
> +#if QSV_HAVE_HE
> +    mfxExtHyperModeParam exthypermodeparam;
> +#endif
>  #if QSV_HAVE_OPAQUE
>      mfxExtOpaqueSurfaceAlloc opaque_alloc;
>      mfxFrameSurface1       **opaque_surfaces;
> @@ -180,7 +192,7 @@ typedef struct QSVEncContext {
>  
>      mfxExtVideoSignalInfo extvsi;
>  
> -    mfxExtBuffer  *extparam_internal[5 + (QSV_HAVE_MF * 2) +
> QSV_HAVE_EXT_AV1_PARAM * 2];
> +    mfxExtBuffer  *extparam_internal[5 + (QSV_HAVE_MF * 2) +
> (QSV_HAVE_EXT_AV1_PARAM * 2) + QSV_HAVE_HE];
>      int         nb_extparam_internal;
>  
>      mfxExtBuffer **extparam;
> @@ -255,6 +267,7 @@ typedef struct QSVEncContext {
>      int co2_idx;
>      int co3_idx;
>      int exthevctiles_idx;
> +    int exthypermodeparam_idx;
>      int vp9_idx;
>  
>      int max_qp_i;
> @@ -299,6 +312,8 @@ typedef struct QSVEncContext {
>      // This is used for SEI Timing reset
>      int old_pic_timing_sei;
>      int skip_frame;
> +    // This is used for Hyper Encode
> +    int dual_gfx;
>  } QSVEncContext;
>  
>  int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q);
> diff --git a/libavcodec/qsvenc_h264.c b/libavcodec/qsvenc_h264.c
> index 0ff7356346..bac337198e 100644
> --- a/libavcodec/qsvenc_h264.c
> +++ b/libavcodec/qsvenc_h264.c
> @@ -117,6 +117,9 @@ static const AVOption options[] = {
>      QSV_OPTION_SCENARIO
>      QSV_OPTION_AVBR
>      QSV_OPTION_SKIP_FRAME
> +#if QSV_HAVE_HE
> +    QSV_HE_OPTIONS
> +#endif
>  
>      { "cavlc",          "Enable
> CAVLC",                           OFFSET(qsv.cavlc),          AV_OPT_TYPE_BOOL
> , { .i64 = 0 },   0,          1, VE },
>  #if QSV_HAVE_VCM
> diff --git a/libavcodec/qsvenc_hevc.c b/libavcodec/qsvenc_hevc.c
> index e042263bf5..5e23ca9647 100644
> --- a/libavcodec/qsvenc_hevc.c
> +++ b/libavcodec/qsvenc_hevc.c
> @@ -318,6 +318,9 @@ static const AVOption options[] = {
>      QSV_OPTION_SCENARIO
>      QSV_OPTION_AVBR
>      QSV_OPTION_SKIP_FRAME
> +#if QSV_HAVE_HE
> +    QSV_HE_OPTIONS
> +#endif
>  
>      { "idr_interval", "Distance (in I-frames) between IDR frames",
> OFFSET(qsv.idr_interval), AV_OPT_TYPE_INT, { .i64 = 0 }, -1, INT_MAX, VE,
> "idr_interval" },
>      { "begin_only", "Output an IDR-frame only at the beginning of the
> stream", 0, AV_OPT_TYPE_CONST, { .i64 = -1 }, 0, 0, VE, "idr_interval" },

LGTM, will apply.

-Haihao