diff mbox series

[FFmpeg-devel,v4] libavcodec/qsvenc: add ROI support to qsv encoder

Message ID 20220608050248.3669348-1-wenbin.chen@intel.com
State Accepted
Commit 97141ffeec803c448d81ee4a53cfa2355f79f7ec
Headers show
Series [FFmpeg-devel,v4] libavcodec/qsvenc: add ROI support to qsv encoder | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_armv7_RPi4 success Make finished
andriy/make_fate_armv7_RPi4 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Wenbin Chen June 8, 2022, 5:02 a.m. UTC
Use The mfxEncoderCtrl parameter to enable ROI. Get side data
"AVRegionOfInterest" and use it to configure "mfxExtEncoderROI" which is
the MediaSDK's ROI configuration.

Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
---
 libavcodec/qsv_internal.h |  4 ++
 libavcodec/qsvenc.c       | 85 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 89 insertions(+)

Comments

Xiang, Haihao June 10, 2022, 3:36 a.m. UTC | #1
On Wed, 2022-06-08 at 13:02 +0800, Wenbin Chen wrote:
> Use The mfxEncoderCtrl parameter to enable ROI. Get side data
> "AVRegionOfInterest" and use it to configure "mfxExtEncoderROI" which is
> the MediaSDK's ROI configuration.
> 
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> ---
>  libavcodec/qsv_internal.h |  4 ++
>  libavcodec/qsvenc.c       | 85 +++++++++++++++++++++++++++++++++++++++
>  2 files changed, 89 insertions(+)
> 
> diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> index e2aecdcbd6..8131acdae9 100644
> --- a/libavcodec/qsv_internal.h
> +++ b/libavcodec/qsv_internal.h
> @@ -51,6 +51,9 @@
>  #define ASYNC_DEPTH_DEFAULT 4       // internal parallelism
>  
>  #define QSV_MAX_ENC_PAYLOAD 2       // # of mfxEncodeCtrl payloads supported
> +#define QSV_MAX_ENC_EXTPARAM 2
> +
> +#define QSV_MAX_ROI_NUM 256
>  
>  #define QSV_MAX_FRAME_EXT_PARAMS 4
>  
> @@ -83,6 +86,7 @@ typedef struct QSVFrame {
>      int num_ext_params;
>  
>      mfxPayload *payloads[QSV_MAX_ENC_PAYLOAD]; ///< used for enc_ctrl.Payload
> +    mfxExtBuffer *extparam[QSV_MAX_ENC_EXTPARAM]; ///< used for
> enc_ctrl.ExtParam
>  
>      int queued;
>      int used;
> diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
> index 03e9e5523d..902bada55b 100644
> --- a/libavcodec/qsvenc.c
> +++ b/libavcodec/qsvenc.c
> @@ -1390,15 +1390,29 @@ static void free_encoder_ctrl_payloads(mfxEncodeCtrl*
> enc_ctrl)
>      }
>  }
>  
> +static void free_encoder_ctrl_extparam(mfxEncodeCtrl* enc_ctrl)
> +{
> +    if (enc_ctrl) {
> +        int i;
> +        for (i = 0; i < enc_ctrl->NumExtParam && i < QSV_MAX_ENC_EXTPARAM;
> i++) {
> +            if (enc_ctrl->ExtParam[i])
> +                av_freep(&(enc_ctrl->ExtParam[i]));
> +        }
> +        enc_ctrl->NumExtParam = 0;
> +    }
> +}
> +
>  static void clear_unused_frames(QSVEncContext *q)
>  {
>      QSVFrame *cur = q->work_frames;
>      while (cur) {
>          if (cur->used && !cur->surface.Data.Locked) {
>              free_encoder_ctrl_payloads(&cur->enc_ctrl);
> +            free_encoder_ctrl_extparam(&cur->enc_ctrl);
>              //do not reuse enc_ctrl from previous frame
>              memset(&cur->enc_ctrl, 0, sizeof(cur->enc_ctrl));
>              cur->enc_ctrl.Payload = cur->payloads;
> +            cur->enc_ctrl.ExtParam = cur->extparam;
>              if (cur->frame->format == AV_PIX_FMT_QSV) {
>                  av_frame_unref(cur->frame);
>              }
> @@ -1436,6 +1450,7 @@ static int get_free_frame(QSVEncContext *q, QSVFrame
> **f)
>          return AVERROR(ENOMEM);
>      }
>      frame->enc_ctrl.Payload = frame->payloads;
> +    frame->enc_ctrl.ExtParam = frame->extparam;
>      *last = frame;
>  
>      *f = frame;
> @@ -1537,6 +1552,67 @@ static void print_interlace_msg(AVCodecContext *avctx,
> QSVEncContext *q)
>      }
>  }
>  
> +static int set_roi_encode_ctrl(AVCodecContext *avctx, const AVFrame *frame,
> +                               mfxEncodeCtrl *enc_ctrl)
> +{
> +    AVFrameSideData *sd = NULL;
> +    int mb_size;
> +
> +    if (avctx->codec_id == AV_CODEC_ID_H264)
> +        mb_size = 16;
> +    else if (avctx->codec_id == AV_CODEC_ID_H265)
> +        mb_size = 32;
> +    else
> +        return 0;
> +
> +    if (frame)
> +        sd = av_frame_get_side_data(frame,
> AV_FRAME_DATA_REGIONS_OF_INTEREST);
> +
> +    if (sd) {
> +        mfxExtEncoderROI *enc_roi = NULL;
> +        AVRegionOfInterest *roi;
> +        uint32_t roi_size;
> +        int nb_roi, i;
> +
> +        roi = (AVRegionOfInterest *)sd->data;
> +        roi_size = roi->self_size;
> +        if (!roi_size || sd->size % roi_size) {
> +            av_log(avctx, AV_LOG_ERROR, "Invalid ROI Data.\n");
> +            return AVERROR(EINVAL);
> +        }
> +        nb_roi = sd->size / roi_size;
> +        if (nb_roi > QSV_MAX_ROI_NUM) {
> +            av_log(avctx, AV_LOG_WARNING, "More ROIs set than "
> +                    "supported by driver (%d > %d).\n",
> +                    nb_roi, QSV_MAX_ROI_NUM);
> +            nb_roi = QSV_MAX_ROI_NUM;
> +        }
> +
> +        enc_roi = av_mallocz(sizeof(*enc_roi));
> +        if (!enc_roi)
> +            return AVERROR(ENOMEM);
> +        enc_roi->Header.BufferId = MFX_EXTBUFF_ENCODER_ROI;
> +        enc_roi->Header.BufferSz = sizeof(*enc_roi);
> +        enc_roi->NumROI  = nb_roi;
> +        enc_roi->ROIMode = MFX_ROI_MODE_QP_DELTA;
> +        for (i = 0; i < nb_roi; i++) {
> +            roi = (AVRegionOfInterest *)(sd->data + roi_size * i);
> +            enc_roi->ROI[i].Top    = FFALIGN(roi->top, mb_size);
> +            enc_roi->ROI[i].Bottom = FFALIGN(roi->bottom, mb_size);
> +            enc_roi->ROI[i].Left   = FFALIGN(roi->left, mb_size);
> +            enc_roi->ROI[i].Right  = FFALIGN(roi->right, mb_size);
> +            enc_roi->ROI[i].DeltaQP =
> +                roi->qoffset.num * 51 / roi->qoffset.den;
> +            av_log(avctx, AV_LOG_DEBUG, "ROI: (%d,%d)-(%d,%d) -> %+d.\n",
> +                   roi->top, roi->left, roi->bottom, roi->right,
> +                   enc_roi->ROI[i].DeltaQP);
> +        }
> +        enc_ctrl->ExtParam[enc_ctrl->NumExtParam] = (mfxExtBuffer *)enc_roi;
> +        enc_ctrl->NumExtParam++;
> +    }
> +    return 0;
> +}
> +
>  static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
>                          const AVFrame *frame)
>  {
> @@ -1599,6 +1675,14 @@ static int encode_frame(AVCodecContext *avctx,
> QSVEncContext *q,
>          q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
>      }
>  
> +    if ((avctx->codec_id == AV_CODEC_ID_H264 ||
> +         avctx->codec_id == AV_CODEC_ID_H265) &&
> +         enc_ctrl && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 8)) {
> +        ret = set_roi_encode_ctrl(avctx, frame, enc_ctrl);
> +        if (ret < 0)
> +            goto free;
> +    }
> +
>      pkt.sync = av_mallocz(sizeof(*pkt.sync));
>      if (!pkt.sync)
>          goto nomem;
> @@ -1721,6 +1805,7 @@ int ff_qsv_enc_close(AVCodecContext *avctx,
> QSVEncContext *q)
>      while (cur) {
>          q->work_frames = cur->next;
>          av_frame_free(&cur->frame);
> +        free_encoder_ctrl_extparam(&cur->enc_ctrl);
>          free_encoder_ctrl_payloads(&cur->enc_ctrl);
>          av_freep(&cur);
>          cur = q->work_frames;

LGTM, will aplly

-Haihao
diff mbox series

Patch

diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
index e2aecdcbd6..8131acdae9 100644
--- a/libavcodec/qsv_internal.h
+++ b/libavcodec/qsv_internal.h
@@ -51,6 +51,9 @@ 
 #define ASYNC_DEPTH_DEFAULT 4       // internal parallelism
 
 #define QSV_MAX_ENC_PAYLOAD 2       // # of mfxEncodeCtrl payloads supported
+#define QSV_MAX_ENC_EXTPARAM 2
+
+#define QSV_MAX_ROI_NUM 256
 
 #define QSV_MAX_FRAME_EXT_PARAMS 4
 
@@ -83,6 +86,7 @@  typedef struct QSVFrame {
     int num_ext_params;
 
     mfxPayload *payloads[QSV_MAX_ENC_PAYLOAD]; ///< used for enc_ctrl.Payload
+    mfxExtBuffer *extparam[QSV_MAX_ENC_EXTPARAM]; ///< used for enc_ctrl.ExtParam
 
     int queued;
     int used;
diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c
index 03e9e5523d..902bada55b 100644
--- a/libavcodec/qsvenc.c
+++ b/libavcodec/qsvenc.c
@@ -1390,15 +1390,29 @@  static void free_encoder_ctrl_payloads(mfxEncodeCtrl* enc_ctrl)
     }
 }
 
+static void free_encoder_ctrl_extparam(mfxEncodeCtrl* enc_ctrl)
+{
+    if (enc_ctrl) {
+        int i;
+        for (i = 0; i < enc_ctrl->NumExtParam && i < QSV_MAX_ENC_EXTPARAM; i++) {
+            if (enc_ctrl->ExtParam[i])
+                av_freep(&(enc_ctrl->ExtParam[i]));
+        }
+        enc_ctrl->NumExtParam = 0;
+    }
+}
+
 static void clear_unused_frames(QSVEncContext *q)
 {
     QSVFrame *cur = q->work_frames;
     while (cur) {
         if (cur->used && !cur->surface.Data.Locked) {
             free_encoder_ctrl_payloads(&cur->enc_ctrl);
+            free_encoder_ctrl_extparam(&cur->enc_ctrl);
             //do not reuse enc_ctrl from previous frame
             memset(&cur->enc_ctrl, 0, sizeof(cur->enc_ctrl));
             cur->enc_ctrl.Payload = cur->payloads;
+            cur->enc_ctrl.ExtParam = cur->extparam;
             if (cur->frame->format == AV_PIX_FMT_QSV) {
                 av_frame_unref(cur->frame);
             }
@@ -1436,6 +1450,7 @@  static int get_free_frame(QSVEncContext *q, QSVFrame **f)
         return AVERROR(ENOMEM);
     }
     frame->enc_ctrl.Payload = frame->payloads;
+    frame->enc_ctrl.ExtParam = frame->extparam;
     *last = frame;
 
     *f = frame;
@@ -1537,6 +1552,67 @@  static void print_interlace_msg(AVCodecContext *avctx, QSVEncContext *q)
     }
 }
 
+static int set_roi_encode_ctrl(AVCodecContext *avctx, const AVFrame *frame,
+                               mfxEncodeCtrl *enc_ctrl)
+{
+    AVFrameSideData *sd = NULL;
+    int mb_size;
+
+    if (avctx->codec_id == AV_CODEC_ID_H264)
+        mb_size = 16;
+    else if (avctx->codec_id == AV_CODEC_ID_H265)
+        mb_size = 32;
+    else
+        return 0;
+
+    if (frame)
+        sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
+
+    if (sd) {
+        mfxExtEncoderROI *enc_roi = NULL;
+        AVRegionOfInterest *roi;
+        uint32_t roi_size;
+        int nb_roi, i;
+
+        roi = (AVRegionOfInterest *)sd->data;
+        roi_size = roi->self_size;
+        if (!roi_size || sd->size % roi_size) {
+            av_log(avctx, AV_LOG_ERROR, "Invalid ROI Data.\n");
+            return AVERROR(EINVAL);
+        }
+        nb_roi = sd->size / roi_size;
+        if (nb_roi > QSV_MAX_ROI_NUM) {
+            av_log(avctx, AV_LOG_WARNING, "More ROIs set than "
+                    "supported by driver (%d > %d).\n",
+                    nb_roi, QSV_MAX_ROI_NUM);
+            nb_roi = QSV_MAX_ROI_NUM;
+        }
+
+        enc_roi = av_mallocz(sizeof(*enc_roi));
+        if (!enc_roi)
+            return AVERROR(ENOMEM);
+        enc_roi->Header.BufferId = MFX_EXTBUFF_ENCODER_ROI;
+        enc_roi->Header.BufferSz = sizeof(*enc_roi);
+        enc_roi->NumROI  = nb_roi;
+        enc_roi->ROIMode = MFX_ROI_MODE_QP_DELTA;
+        for (i = 0; i < nb_roi; i++) {
+            roi = (AVRegionOfInterest *)(sd->data + roi_size * i);
+            enc_roi->ROI[i].Top    = FFALIGN(roi->top, mb_size);
+            enc_roi->ROI[i].Bottom = FFALIGN(roi->bottom, mb_size);
+            enc_roi->ROI[i].Left   = FFALIGN(roi->left, mb_size);
+            enc_roi->ROI[i].Right  = FFALIGN(roi->right, mb_size);
+            enc_roi->ROI[i].DeltaQP =
+                roi->qoffset.num * 51 / roi->qoffset.den;
+            av_log(avctx, AV_LOG_DEBUG, "ROI: (%d,%d)-(%d,%d) -> %+d.\n",
+                   roi->top, roi->left, roi->bottom, roi->right,
+                   enc_roi->ROI[i].DeltaQP);
+        }
+        enc_ctrl->ExtParam[enc_ctrl->NumExtParam] = (mfxExtBuffer *)enc_roi;
+        enc_ctrl->NumExtParam++;
+    }
+    return 0;
+}
+
 static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
                         const AVFrame *frame)
 {
@@ -1599,6 +1675,14 @@  static int encode_frame(AVCodecContext *avctx, QSVEncContext *q,
         q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl);
     }
 
+    if ((avctx->codec_id == AV_CODEC_ID_H264 ||
+         avctx->codec_id == AV_CODEC_ID_H265) &&
+         enc_ctrl && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 8)) {
+        ret = set_roi_encode_ctrl(avctx, frame, enc_ctrl);
+        if (ret < 0)
+            goto free;
+    }
+
     pkt.sync = av_mallocz(sizeof(*pkt.sync));
     if (!pkt.sync)
         goto nomem;
@@ -1721,6 +1805,7 @@  int ff_qsv_enc_close(AVCodecContext *avctx, QSVEncContext *q)
     while (cur) {
         q->work_frames = cur->next;
         av_frame_free(&cur->frame);
+        free_encoder_ctrl_extparam(&cur->enc_ctrl);
         free_encoder_ctrl_payloads(&cur->enc_ctrl);
         av_freep(&cur);
         cur = q->work_frames;