diff mbox

[FFmpeg-devel] qsvenc: Add ROI encoding support for h264_qsv and hevc_qsv

Message ID 1577271735-9180-1-git-send-email-yejun.guo@intel.com
State New
Headers show

Commit Message

Guo, Yejun Dec. 25, 2019, 11:02 a.m. UTC 
command examples:
./ffmpeg  -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:0.5 -c:v hevc_qsv  -b:v 5M -y qsv.b5m.roi.h265
./ffmpeg  -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:-0.5 -c:v h264_qsv -mfmode 1 -b:v 4M -y qsv.b4m.roi.h264

Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
---
 libavcodec/qsvenc.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 libavcodec/qsvenc.h |  12 +++++-
 2 files changed, 124 insertions(+), 1 deletion(-)

Comments

Zhong Li Dec. 25, 2019, 3:02 p.m. UTC | #1 
Guo, Yejun <yejun.guo@intel.com> 于2019年12月25日周三 下午7:10写道：
>
> command examples:
> ./ffmpeg  -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:0.5 -c:v hevc_qsv  -b:v 5M -y qsv.b5m.roi.h265
> ./ffmpeg  -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:-0.5 -c:v h264_qsv -mfmode 1 -b:v 4M -y qsv.b4m.roi.h264
>
> Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
> ---
>  libavcodec/qsvenc.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>  libavcodec/qsvenc.h |  12 +++++-
>  2 files changed, 124 insertions(+), 1 deletion(-)
>
> diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
> index 9e41650..aa42c9b 100644
> --- a/libavcodec/qsvenc.c
> +++ b/libavcodec/qsvenc.c
> @@ -482,6 +482,7 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q)
>      int target_bitrate_kbps, max_bitrate_kbps, brc_param_multiplier;
>      int buffer_size_in_kilobytes, initial_delay_in_kilobytes;
>      int ret;
> +    av_unused int mfe_enabled = 0;
>
>      ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
>      if (ret < 0)
> @@ -755,6 +756,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
>                  q->extmfp.Header.BufferSz     = sizeof(q->extmfp);
>
>                  q->extmfp.MFMode = q->mfmode;
> +                mfe_enabled = q->mfmode != MFX_MF_DISABLED;
>                  av_log(avctx,AV_LOG_VERBOSE,"MFMode:%d\n", q->extmfp.MFMode);
>                  q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extmfp;
>              }
> @@ -790,6 +792,25 @@ FF_ENABLE_DEPRECATION_WARNINGS
>      }
>  #endif
>
> +#if QSV_HAVE_ROI_ENCODING
> +    // mfe and roi could not be enabled together, they even could not be queried together,
> +    // otherwise, gpu hang.
This should be fixed in MSDK instead of workaround here.

> +    if (!mfe_enabled) {
> +        q->extroi.Header.BufferId = MFX_EXTBUFF_ENCODER_ROI;
> +        q->extroi.Header.BufferSz = sizeof(q->extroi);
> +        q->extroi.ROIMode = MFX_ROI_MODE_QP_DELTA;
> +        // 256 to query the maximum supported value
> +        q->extroi.NumROI = 256;
> +        // due to the requirement of msdk, we must set non_empty rect for query
> +        for (int i = 0; i < sizeof(q->extroi.ROI)/sizeof(q->extroi.ROI[0]); ++i) {

Should be good to use FF_ARRAY_ELEMS()?

> +            q->extroi.ROI[i].Right  = 16;
> +            q->extroi.ROI[i].Bottom = 16;
> +        }
> +        q->roi_index_in_internal_buffer = q->nb_extparam_internal;
> +        q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extroi;
> +    }
> +#endif
> +
>      if (!check_enc_param(avctx,q)) {
>          av_log(avctx, AV_LOG_ERROR,
>                 "some encoding parameters are not supported by the QSV "
> @@ -1178,6 +1199,10 @@ int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q)
>                                    "Error querying encoder params");
>      }
>
> +#if QSV_HAVE_ROI_ENCODING
> +    q->roi_max_regions = q->extroi.NumROI;
> +#endif
> +
>      ret = MFXVideoENCODE_QueryIOSurf(q->session, &q->param, &q->req);
>      if (ret < 0)
>          return ff_qsv_print_error(avctx, ret,
> @@ -1375,6 +1400,88 @@ static void print_interlace_msg(AVCodecContext *avctx, QSVEncContext *q)
>      }
>  }
>
> +static int qsv_encode_set_roi(AVCodecContext *avctx, QSVEncContext *q, const AVFrame *frame, mfxEncodeCtrl *enc_ctrl)
> +{
> +#if QSV_HAVE_ROI_ENCODING
> +    // in qsv, the min_delta_qp is -51, and the max_delta_qp is 51
> +    int qp_range = 51;
This is only true for HEVC and H264, not ture for vp8 and vp9.
Though I see ROI is just supported for HEVC and H264 now.

> +    int nb_roi;
> +    const AVRegionOfInterest *roi;
> +    uint32_t roi_size;
> +
> +    AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
> +    if (!sd) {
> +        // need to reset for the case: with roi side data --> no roi side data
> +        enc_ctrl->NumExtParam = 0;
> +        return 0;
> +    }
> +
> +#if QSV_HAVE_MF
> +    if (avctx->codec_id == AV_CODEC_ID_H264) {
> +        if (q->mfmode != MFX_MF_DISABLED) {
> +            if (!q->roi_warned) {
> +                q->roi_warned = 1;
> +                av_log(avctx, AV_LOG_WARNING, "to enable ROI, "
> +                       "please make sure Multi-Frame Mode is disabled.\n");
> +            }
> +            return 0;
> +        }
> +    }
> +#endif
> +
> +    // msdk currently just supports roi encoding for h264 and h265
> +    if (avctx->codec_id != AV_CODEC_ID_H264 && avctx->codec_id != AV_CODEC_ID_HEVC) {

I belive you should also check MSDK run-time libary version which can
support ROI.

> +        if (!q->roi_warned) {
> +            q->roi_warned = 1;
> +            av_log(avctx, AV_LOG_WARNING, "ROI encoding is only enabled for h264_qsv and hevc_qsv.\n");
> +        }
> +        return 0;
> +    }
> +
> +    roi = (const AVRegionOfInterest*)sd->data;
> +    roi_size = roi->self_size;
> +    if (!roi_size || sd->size % roi_size != 0) {
> +        av_log(avctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n");
> +        return AVERROR(EINVAL);
> +    }
> +
> +    nb_roi = sd->size / roi_size;
> +    if (nb_roi > q->roi_max_regions) {
> +        if (!q->roi_warned) {
> +            av_log(avctx, AV_LOG_WARNING, "More ROIs set than supported (%d > %d).\n",
> +                   nb_roi, q->roi_max_regions);
> +            q->roi_warned = 1;
> +        }
> +        nb_roi = q->roi_max_regions;
> +    }
> +    q->extroi.NumROI = nb_roi;
> +
> +    // For overlapping regions, the first in the array takes priority.
> +    for (int i = 0; i < nb_roi; i++) {
> +        float qoffset;
> +
> +        roi = (const AVRegionOfInterest*)(sd->data + roi_size * i);
> +        if (roi->qoffset.den == 0) {
> +            av_log(avctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must not be zero.\n");
> +            return AVERROR(EINVAL);
> +        }
> +        qoffset = roi->qoffset.num * 1.0f / roi->qoffset.den;
> +        qoffset = av_clipf(qoffset * qp_range, -qp_range, +qp_range);
> +
> +        q->extroi.ROI[i].Left = roi->left;
> +        q->extroi.ROI[i].Right = roi->right;
> +        q->extroi.ROI[i].Top = roi->top;
> +        q->extroi.ROI[i].Bottom = roi->bottom;
> +        q->extroi.ROI[i].DeltaQP = qoffset;
> +    }
> +
> +    enc_ctrl->NumExtParam = 1;
> +    enc_ctrl->ExtParam = &q->extparam_internal[q->roi_index_in_internal_buffer];
> +#endif
> +
> +    return 0;
> +}
> +
>  static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
>                          const AVFrame *frame)
>  {
> @@ -1445,6 +1552,12 @@ static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
>          q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
>      }
>
> +    if (frame) {
> +        ret = qsv_encode_set_roi(avctx, q, frame, &qsv_frame->enc_ctrl);
> +        if (ret < 0)
> +            return ret;
> +    }
> +
>      sync = av_mallocz(sizeof(*sync));
>      if (!sync) {
>          av_freep(&bs);
> diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h
> index 6609171..9c6caf1 100644
> --- a/libavcodec/qsvenc.h
> +++ b/libavcodec/qsvenc.h
> @@ -40,6 +40,7 @@
>
>  #define QSV_HAVE_EXT_HEVC_TILES QSV_VERSION_ATLEAST(1, 13)
>  #define QSV_HAVE_EXT_VP9_PARAM QSV_VERSION_ATLEAST(1, 26)
> +#define QSV_HAVE_ROI_ENCODING QSV_VERSION_ATLEAST(1, 22)
>
>  #define QSV_HAVE_TRELLIS QSV_VERSION_ATLEAST(1, 8)
>  #define QSV_HAVE_MAX_SLICE_SIZE QSV_VERSION_ATLEAST(1, 9)
> @@ -131,12 +132,21 @@ typedef struct QSVEncContext {
>  #if QSV_HAVE_EXT_VP9_PARAM
>      mfxExtVP9Param  extvp9param;
>  #endif
> +#if QSV_HAVE_ROI_ENCODING
> +    mfxExtEncoderROI extroi;
> +    // Maximum number of regions supported
> +    int roi_max_regions;
> +    int roi_index_in_internal_buffer;
> +    // If the driver does not support ROI then warn the first time we
> +    // encounter a frame with ROI side data.
> +    int roi_warned;
> +#endif
>
>      mfxExtOpaqueSurfaceAlloc opaque_alloc;
>      mfxFrameSurface1       **opaque_surfaces;
>      AVBufferRef             *opaque_alloc_buf;
>
> -    mfxExtBuffer  *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2)];
> +    mfxExtBuffer  *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2) + QSV_HAVE_ROI_ENCODING];
>      int         nb_extparam_internal;
>
>      mfxExtBuffer **extparam;
> --
> 2.7.4
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff mbox

Patch

diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
index 9e41650..aa42c9b 100644
--- a/libavcodec/qsvenc.c
+++ b/libavcodec/qsvenc.c
@@ -482,6 +482,7 @@  static int init_video_param(AVCodecContext *avctx, QSVEncContext *q)
     int target_bitrate_kbps, max_bitrate_kbps, brc_param_multiplier;
     int buffer_size_in_kilobytes, initial_delay_in_kilobytes;
     int ret;
+    av_unused int mfe_enabled = 0;
 
     ret = ff_qsv_codec_id_to_mfx(avctx->codec_id);
     if (ret < 0)
@@ -755,6 +756,7 @@  FF_ENABLE_DEPRECATION_WARNINGS
                 q->extmfp.Header.BufferSz     = sizeof(q->extmfp);
 
                 q->extmfp.MFMode = q->mfmode;
+                mfe_enabled = q->mfmode != MFX_MF_DISABLED;
                 av_log(avctx,AV_LOG_VERBOSE,"MFMode:%d\n", q->extmfp.MFMode);
                 q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extmfp;
             }
@@ -790,6 +792,25 @@  FF_ENABLE_DEPRECATION_WARNINGS
     }
 #endif
 
+#if QSV_HAVE_ROI_ENCODING
+    // mfe and roi could not be enabled together, they even could not be queried together,
+    // otherwise, gpu hang.
+    if (!mfe_enabled) {
+        q->extroi.Header.BufferId = MFX_EXTBUFF_ENCODER_ROI;
+        q->extroi.Header.BufferSz = sizeof(q->extroi);
+        q->extroi.ROIMode = MFX_ROI_MODE_QP_DELTA;
+        // 256 to query the maximum supported value
+        q->extroi.NumROI = 256;
+        // due to the requirement of msdk, we must set non_empty rect for query
+        for (int i = 0; i < sizeof(q->extroi.ROI)/sizeof(q->extroi.ROI[0]); ++i) {
+            q->extroi.ROI[i].Right  = 16;
+            q->extroi.ROI[i].Bottom = 16;
+        }
+        q->roi_index_in_internal_buffer = q->nb_extparam_internal;
+        q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extroi;
+    }
+#endif
+
     if (!check_enc_param(avctx,q)) {
         av_log(avctx, AV_LOG_ERROR,
                "some encoding parameters are not supported by the QSV "
@@ -1178,6 +1199,10 @@  int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q)
                                   "Error querying encoder params");
     }
 
+#if QSV_HAVE_ROI_ENCODING
+    q->roi_max_regions = q->extroi.NumROI;
+#endif
+
     ret = MFXVideoENCODE_QueryIOSurf(q->session, &q->param, &q->req);
     if (ret < 0)
         return ff_qsv_print_error(avctx, ret,
@@ -1375,6 +1400,88 @@  static void print_interlace_msg(AVCodecContext *avctx, QSVEncContext *q)
     }
 }
 
+static int qsv_encode_set_roi(AVCodecContext *avctx, QSVEncContext *q, const AVFrame *frame, mfxEncodeCtrl *enc_ctrl)
+{
+#if QSV_HAVE_ROI_ENCODING
+    // in qsv, the min_delta_qp is -51, and the max_delta_qp is 51
+    int qp_range = 51;
+    int nb_roi;
+    const AVRegionOfInterest *roi;
+    uint32_t roi_size;
+
+    AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
+    if (!sd) {
+        // need to reset for the case: with roi side data --> no roi side data
+        enc_ctrl->NumExtParam = 0;
+        return 0;
+    }
+
+#if QSV_HAVE_MF
+    if (avctx->codec_id == AV_CODEC_ID_H264) {
+        if (q->mfmode != MFX_MF_DISABLED) {
+            if (!q->roi_warned) {
+                q->roi_warned = 1;
+                av_log(avctx, AV_LOG_WARNING, "to enable ROI, "
+                       "please make sure Multi-Frame Mode is disabled.\n");
+            }
+            return 0;
+        }
+    }
+#endif
+
+    // msdk currently just supports roi encoding for h264 and h265
+    if (avctx->codec_id != AV_CODEC_ID_H264 && avctx->codec_id != AV_CODEC_ID_HEVC) {
+        if (!q->roi_warned) {
+            q->roi_warned = 1;
+            av_log(avctx, AV_LOG_WARNING, "ROI encoding is only enabled for h264_qsv and hevc_qsv.\n");
+        }
+        return 0;
+    }
+
+    roi = (const AVRegionOfInterest*)sd->data;
+    roi_size = roi->self_size;
+    if (!roi_size || sd->size % roi_size != 0) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n");
+        return AVERROR(EINVAL);
+    }
+
+    nb_roi = sd->size / roi_size;
+    if (nb_roi > q->roi_max_regions) {
+        if (!q->roi_warned) {
+            av_log(avctx, AV_LOG_WARNING, "More ROIs set than supported (%d > %d).\n",
+                   nb_roi, q->roi_max_regions);
+            q->roi_warned = 1;
+        }
+        nb_roi = q->roi_max_regions;
+    }
+    q->extroi.NumROI = nb_roi;
+
+    // For overlapping regions, the first in the array takes priority.
+    for (int i = 0; i < nb_roi; i++) {
+        float qoffset;
+
+        roi = (const AVRegionOfInterest*)(sd->data + roi_size * i);
+        if (roi->qoffset.den == 0) {
+            av_log(avctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must not be zero.\n");
+            return AVERROR(EINVAL);
+        }
+        qoffset = roi->qoffset.num * 1.0f / roi->qoffset.den;
+        qoffset = av_clipf(qoffset * qp_range, -qp_range, +qp_range);
+
+        q->extroi.ROI[i].Left = roi->left;
+        q->extroi.ROI[i].Right = roi->right;
+        q->extroi.ROI[i].Top = roi->top;
+        q->extroi.ROI[i].Bottom = roi->bottom;
+        q->extroi.ROI[i].DeltaQP = qoffset;
+    }
+
+    enc_ctrl->NumExtParam = 1;
+    enc_ctrl->ExtParam = &q->extparam_internal[q->roi_index_in_internal_buffer];
+#endif
+
+    return 0;
+}
+
 static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
                         const AVFrame *frame)
 {
@@ -1445,6 +1552,12 @@  static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
         q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
     }
 
+    if (frame) {
+        ret = qsv_encode_set_roi(avctx, q, frame, &qsv_frame->enc_ctrl);
+        if (ret < 0)
+            return ret;
+    }
+
     sync = av_mallocz(sizeof(*sync));
     if (!sync) {
         av_freep(&bs);
diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h
index 6609171..9c6caf1 100644
--- a/libavcodec/qsvenc.h
+++ b/libavcodec/qsvenc.h
@@ -40,6 +40,7 @@ 
 
 #define QSV_HAVE_EXT_HEVC_TILES QSV_VERSION_ATLEAST(1, 13)
 #define QSV_HAVE_EXT_VP9_PARAM QSV_VERSION_ATLEAST(1, 26)
+#define QSV_HAVE_ROI_ENCODING QSV_VERSION_ATLEAST(1, 22)
 
 #define QSV_HAVE_TRELLIS QSV_VERSION_ATLEAST(1, 8)
 #define QSV_HAVE_MAX_SLICE_SIZE QSV_VERSION_ATLEAST(1, 9)
@@ -131,12 +132,21 @@  typedef struct QSVEncContext {
 #if QSV_HAVE_EXT_VP9_PARAM
     mfxExtVP9Param  extvp9param;
 #endif
+#if QSV_HAVE_ROI_ENCODING
+    mfxExtEncoderROI extroi;
+    // Maximum number of regions supported
+    int roi_max_regions;
+    int roi_index_in_internal_buffer;
+    // If the driver does not support ROI then warn the first time we
+    // encounter a frame with ROI side data.
+    int roi_warned;
+#endif
 
     mfxExtOpaqueSurfaceAlloc opaque_alloc;
     mfxFrameSurface1       **opaque_surfaces;
     AVBufferRef             *opaque_alloc_buf;
 
-    mfxExtBuffer  *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2)];
+    mfxExtBuffer  *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2) + QSV_HAVE_ROI_ENCODING];
     int         nb_extparam_internal;
 
     mfxExtBuffer **extparam;