| Message ID | 20220506054957.2291057-1-wenbin.chen@intel.com |
|---|---|
| State | New |
| Headers | show |
| Series | [FFmpeg-devel] libavcodec/qsvenc: add ROI support to qsv encoder | expand |
| Context | Check | Description |
|---|---|---|
| yinshiyou/make_loongarch64 | success | Make finished |
| yinshiyou/make_fate_loongarch64 | success | Make fate finished |
| andriy/make_x86 | success | Make finished |
| andriy/make_fate_x86 | success | Make fate finished |
On Fri, 2022-05-06 at 13:49 +0800, Wenbin Chen wrote: > Use the mfxEncoderCtrl parameter to enable ROI. Get side data > "AVRegionOfInterest" > from filter "addroi" and use it to configure "mfxExtEncoderROI" which is > the MediaSDK's ROI configuration. > > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> > --- > libavcodec/qsv_internal.h | 4 ++ > libavcodec/qsvenc.c | 88 +++++++++++++++++++++++++++++++++++++++ > 2 files changed, 92 insertions(+) > > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h > index e2aecdcbd6..8131acdae9 100644 > --- a/libavcodec/qsv_internal.h > +++ b/libavcodec/qsv_internal.h > @@ -51,6 +51,9 @@ > #define ASYNC_DEPTH_DEFAULT 4 // internal parallelism > > #define QSV_MAX_ENC_PAYLOAD 2 // # of mfxEncodeCtrl payloads supported > +#define QSV_MAX_ENC_EXTPARAM 2 > + > +#define QSV_MAX_ROI_NUM 256 > > #define QSV_MAX_FRAME_EXT_PARAMS 4 > > @@ -83,6 +86,7 @@ typedef struct QSVFrame { > int num_ext_params; > > mfxPayload *payloads[QSV_MAX_ENC_PAYLOAD]; ///< used for enc_ctrl.Payload > + mfxExtBuffer *extparam[QSV_MAX_ENC_EXTPARAM]; ///< used for > enc_ctrl.ExtParam > > int queued; > int used; > diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c > index fbb22ca436..9abead604b 100644 > --- a/libavcodec/qsvenc.c > +++ b/libavcodec/qsvenc.c > @@ -1501,15 +1501,29 @@ static void free_encoder_ctrl_payloads(mfxEncodeCtrl* > enc_ctrl) > } > } > > +static void free_encoder_ctrl_extparam(mfxEncodeCtrl* enc_ctrl) > +{ > + if (enc_ctrl) { > + int i; > + for (i = 0; i < enc_ctrl->NumExtParam && i < QSV_MAX_ENC_EXTPARAM; > i++) { > + if (enc_ctrl->ExtParam[i]) > + av_freep(&(enc_ctrl->ExtParam[i])); > + } > + enc_ctrl->NumExtParam = 0; > + } > +} > + > static void clear_unused_frames(QSVEncContext *q) > { > QSVFrame *cur = q->work_frames; > while (cur) { > if (cur->used && !cur->surface.Data.Locked) { > free_encoder_ctrl_payloads(&cur->enc_ctrl); > + free_encoder_ctrl_extparam(&cur->enc_ctrl); > //do not reuse enc_ctrl from previous frame > memset(&cur->enc_ctrl, 0, sizeof(cur->enc_ctrl)); > cur->enc_ctrl.Payload = cur->payloads; > + cur->enc_ctrl.ExtParam = cur->extparam; > if (cur->frame->format == AV_PIX_FMT_QSV) { > av_frame_unref(cur->frame); > } > @@ -1547,6 +1561,7 @@ static int get_free_frame(QSVEncContext *q, QSVFrame > **f) > return AVERROR(ENOMEM); > } > frame->enc_ctrl.Payload = frame->payloads; > + frame->enc_ctrl.ExtParam = frame->extparam; > *last = frame; > > *f = frame; > @@ -1648,6 +1663,70 @@ static void print_interlace_msg(AVCodecContext *avctx, > QSVEncContext *q) > } > } > > +static int set_roi_encode_ctrl(AVCodecContext *avctx, const AVFrame *frame, > + mfxEncodeCtrl *enc_ctrl) > +{ > +#if QSV_VERSION_ATLEAST(1, 22) This check can be removed now because the minimal version is 1.28 > + AVFrameSideData *sd = NULL; > + int mb_size; > + > + if (avctx->codec_id == AV_CODEC_ID_H264) { > + mb_size = 16; > + } else if (avctx->codec_id == AV_CODEC_ID_H265) { > + mb_size = 32; > + } else { > + return 0; > + } > + > + if (frame) > + sd = av_frame_get_side_data(frame, > AV_FRAME_DATA_REGIONS_OF_INTEREST); > + > + if (sd) { > + mfxExtEncoderROI *enc_roi = NULL; > + AVRegionOfInterest *roi; > + uint32_t roi_size; > + int nb_roi, i; > + > + roi = (AVRegionOfInterest *)sd->data; > + roi_size = roi->self_size; > + if (!roi_size || sd->size % roi_size) { > + av_log(avctx, AV_LOG_ERROR, "Invalid ROI Data.\n"); > + return AVERROR(EINVAL); > + } > + nb_roi = sd->size / roi_size; > + if (nb_roi > QSV_MAX_ROI_NUM) { > + av_log(avctx, AV_LOG_WARNING, "More ROIs set than " > + "supported by driver (%d > %d).\n", > + nb_roi, QSV_MAX_ROI_NUM); > + nb_roi = QSV_MAX_ROI_NUM; > + } > + > + enc_roi = av_mallocz(sizeof(*enc_roi)); > + if (!enc_roi) > + return AVERROR(ENOMEM); > + enc_roi->Header.BufferId = MFX_EXTBUFF_ENCODER_ROI; > + enc_roi->Header.BufferSz = sizeof(*enc_roi); > + enc_roi->NumROI = nb_roi; > + enc_roi->ROIMode = MFX_ROI_MODE_QP_DELTA; > + for (i = 0; i < nb_roi; i++) { > + roi = (AVRegionOfInterest *)(sd->data + roi_size * i); > + enc_roi->ROI[i].Top = FFALIGN(roi->top, mb_size); > + enc_roi->ROI[i].Bottom = FFALIGN(roi->bottom, mb_size); > + enc_roi->ROI[i].Left = FFALIGN(roi->left, mb_size); > + enc_roi->ROI[i].Right = FFALIGN(roi->right, mb_size); > + enc_roi->ROI[i].DeltaQP = > + roi->qoffset.num * 51 / roi->qoffset.den; > + av_log(avctx, AV_LOG_DEBUG, "ROI: (%d,%d)-(%d,%d) -> %+d.\n", > + roi->top, roi->left, roi->bottom, roi->right, > + enc_roi->ROI[i].DeltaQP); > + } > + enc_ctrl->ExtParam[enc_ctrl->NumExtParam] = (mfxExtBuffer *)enc_roi; You should ensure the runtime supports MFX_EXTBUFF_ENCODER_ROI when adding MFX_EXTBUFF_ENCODER_ROI buffer to enc_ctrl->ExtParam[]. Thanks Haihao > + enc_ctrl->NumExtParam++; > + } > +#endif > + return 0; > +} > + > static int encode_frame(AVCodecContext *avctx, QSVEncContext *q, > const AVFrame *frame) > { > @@ -1714,6 +1793,14 @@ static int encode_frame(AVCodecContext *avctx, > QSVEncContext *q, > q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl); > } > > + if ((avctx->codec_id == AV_CODEC_ID_H264 || > + avctx->codec_id == AV_CODEC_ID_H265) && > + enc_ctrl) { > + ret = set_roi_encode_ctrl(avctx, frame, enc_ctrl); > + if (ret < 0) > + goto free; > + } > + > pkt.sync = av_mallocz(sizeof(*pkt.sync)); > if (!pkt.sync) > goto nomem; > @@ -1842,6 +1929,7 @@ int ff_qsv_enc_close(AVCodecContext *avctx, > QSVEncContext *q) > while (cur) { > q->work_frames = cur->next; > av_frame_free(&cur->frame); > + free_encoder_ctrl_extparam(&cur->enc_ctrl); > free_encoder_ctrl_payloads(&cur->enc_ctrl); > av_freep(&cur); > cur = q->work_frames;
> On Fri, 2022-05-06 at 13:49 +0800, Wenbin Chen wrote: > > Use the mfxEncoderCtrl parameter to enable ROI. Get side data > > "AVRegionOfInterest" > > from filter "addroi" and use it to configure "mfxExtEncoderROI" which is > > the MediaSDK's ROI configuration. > > > > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> > > --- > > libavcodec/qsv_internal.h | 4 ++ > > libavcodec/qsvenc.c | 88 > +++++++++++++++++++++++++++++++++++++++ > > 2 files changed, 92 insertions(+) > > > > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h > > index e2aecdcbd6..8131acdae9 100644 > > --- a/libavcodec/qsv_internal.h > > +++ b/libavcodec/qsv_internal.h > > @@ -51,6 +51,9 @@ > > #define ASYNC_DEPTH_DEFAULT 4 // internal parallelism > > > > #define QSV_MAX_ENC_PAYLOAD 2 // # of mfxEncodeCtrl payloads > supported > > +#define QSV_MAX_ENC_EXTPARAM 2 > > + > > +#define QSV_MAX_ROI_NUM 256 > > > > #define QSV_MAX_FRAME_EXT_PARAMS 4 > > > > @@ -83,6 +86,7 @@ typedef struct QSVFrame { > > int num_ext_params; > > > > mfxPayload *payloads[QSV_MAX_ENC_PAYLOAD]; ///< used for > enc_ctrl.Payload > > + mfxExtBuffer *extparam[QSV_MAX_ENC_EXTPARAM]; ///< used for > > enc_ctrl.ExtParam > > > > int queued; > > int used; > > diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c > > index fbb22ca436..9abead604b 100644 > > --- a/libavcodec/qsvenc.c > > +++ b/libavcodec/qsvenc.c > > @@ -1501,15 +1501,29 @@ static void > free_encoder_ctrl_payloads(mfxEncodeCtrl* > > enc_ctrl) > > } > > } > > > > +static void free_encoder_ctrl_extparam(mfxEncodeCtrl* enc_ctrl) > > +{ > > + if (enc_ctrl) { > > + int i; > > + for (i = 0; i < enc_ctrl->NumExtParam && i < > QSV_MAX_ENC_EXTPARAM; > > i++) { > > + if (enc_ctrl->ExtParam[i]) > > + av_freep(&(enc_ctrl->ExtParam[i])); > > + } > > + enc_ctrl->NumExtParam = 0; > > + } > > +} > > + > > static void clear_unused_frames(QSVEncContext *q) > > { > > QSVFrame *cur = q->work_frames; > > while (cur) { > > if (cur->used && !cur->surface.Data.Locked) { > > free_encoder_ctrl_payloads(&cur->enc_ctrl); > > + free_encoder_ctrl_extparam(&cur->enc_ctrl); > > //do not reuse enc_ctrl from previous frame > > memset(&cur->enc_ctrl, 0, sizeof(cur->enc_ctrl)); > > cur->enc_ctrl.Payload = cur->payloads; > > + cur->enc_ctrl.ExtParam = cur->extparam; > > if (cur->frame->format == AV_PIX_FMT_QSV) { > > av_frame_unref(cur->frame); > > } > > @@ -1547,6 +1561,7 @@ static int get_free_frame(QSVEncContext *q, > QSVFrame > > **f) > > return AVERROR(ENOMEM); > > } > > frame->enc_ctrl.Payload = frame->payloads; > > + frame->enc_ctrl.ExtParam = frame->extparam; > > *last = frame; > > > > *f = frame; > > @@ -1648,6 +1663,70 @@ static void print_interlace_msg(AVCodecContext > *avctx, > > QSVEncContext *q) > > } > > } > > > > +static int set_roi_encode_ctrl(AVCodecContext *avctx, const AVFrame > *frame, > > + mfxEncodeCtrl *enc_ctrl) > > +{ > > +#if QSV_VERSION_ATLEAST(1, 22) > > This check can be removed now because the minimal version is 1.28 > > > + AVFrameSideData *sd = NULL; > > + int mb_size; > > + > > + if (avctx->codec_id == AV_CODEC_ID_H264) { > > + mb_size = 16; > > + } else if (avctx->codec_id == AV_CODEC_ID_H265) { > > + mb_size = 32; > > + } else { > > + return 0; > > + } > > + > > + if (frame) > > + sd = av_frame_get_side_data(frame, > > AV_FRAME_DATA_REGIONS_OF_INTEREST); > > + > > + if (sd) { > > + mfxExtEncoderROI *enc_roi = NULL; > > + AVRegionOfInterest *roi; > > + uint32_t roi_size; > > + int nb_roi, i; > > + > > + roi = (AVRegionOfInterest *)sd->data; > > + roi_size = roi->self_size; > > + if (!roi_size || sd->size % roi_size) { > > + av_log(avctx, AV_LOG_ERROR, "Invalid ROI Data.\n"); > > + return AVERROR(EINVAL); > > + } > > + nb_roi = sd->size / roi_size; > > + if (nb_roi > QSV_MAX_ROI_NUM) { > > + av_log(avctx, AV_LOG_WARNING, "More ROIs set than " > > + "supported by driver (%d > %d).\n", > > + nb_roi, QSV_MAX_ROI_NUM); > > + nb_roi = QSV_MAX_ROI_NUM; > > + } > > + > > + enc_roi = av_mallocz(sizeof(*enc_roi)); > > + if (!enc_roi) > > + return AVERROR(ENOMEM); > > + enc_roi->Header.BufferId = MFX_EXTBUFF_ENCODER_ROI; > > + enc_roi->Header.BufferSz = sizeof(*enc_roi); > > + enc_roi->NumROI = nb_roi; > > + enc_roi->ROIMode = MFX_ROI_MODE_QP_DELTA; > > + for (i = 0; i < nb_roi; i++) { > > + roi = (AVRegionOfInterest *)(sd->data + roi_size * i); > > + enc_roi->ROI[i].Top = FFALIGN(roi->top, mb_size); > > + enc_roi->ROI[i].Bottom = FFALIGN(roi->bottom, mb_size); > > + enc_roi->ROI[i].Left = FFALIGN(roi->left, mb_size); > > + enc_roi->ROI[i].Right = FFALIGN(roi->right, mb_size); > > + enc_roi->ROI[i].DeltaQP = > > + roi->qoffset.num * 51 / roi->qoffset.den; > > + av_log(avctx, AV_LOG_DEBUG, "ROI: (%d,%d)-(%d,%d) -> %+d.\n", > > + roi->top, roi->left, roi->bottom, roi->right, > > + enc_roi->ROI[i].DeltaQP); > > + } > > + enc_ctrl->ExtParam[enc_ctrl->NumExtParam] = (mfxExtBuffer > *)enc_roi; > > > You should ensure the runtime supports MFX_EXTBUFF_ENCODER_ROI > when adding > MFX_EXTBUFF_ENCODER_ROI buffer to enc_ctrl->ExtParam[]. > > Thanks > Haihao Thanks for review. I will update the PR. I will add runtime check before call set_roi_encode_ctrl(). Thanks Wenbin > > > + enc_ctrl->NumExtParam++; > > + } > > +#endif > > + return 0; > > +} > > + > > static int encode_frame(AVCodecContext *avctx, QSVEncContext *q, > > const AVFrame *frame) > > { > > @@ -1714,6 +1793,14 @@ static int encode_frame(AVCodecContext > *avctx, > > QSVEncContext *q, > > q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl); > > } > > > > + if ((avctx->codec_id == AV_CODEC_ID_H264 || > > + avctx->codec_id == AV_CODEC_ID_H265) && > > + enc_ctrl) { > > + ret = set_roi_encode_ctrl(avctx, frame, enc_ctrl); > > + if (ret < 0) > > + goto free; > > + } > > + > > pkt.sync = av_mallocz(sizeof(*pkt.sync)); > > if (!pkt.sync) > > goto nomem; > > @@ -1842,6 +1929,7 @@ int ff_qsv_enc_close(AVCodecContext *avctx, > > QSVEncContext *q) > > while (cur) { > > q->work_frames = cur->next; > > av_frame_free(&cur->frame); > > + free_encoder_ctrl_extparam(&cur->enc_ctrl); > > free_encoder_ctrl_payloads(&cur->enc_ctrl); > > av_freep(&cur); > > cur = q->work_frames; > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h index e2aecdcbd6..8131acdae9 100644 --- a/libavcodec/qsv_internal.h +++ b/libavcodec/qsv_internal.h @@ -51,6 +51,9 @@ #define ASYNC_DEPTH_DEFAULT 4 // internal parallelism #define QSV_MAX_ENC_PAYLOAD 2 // # of mfxEncodeCtrl payloads supported +#define QSV_MAX_ENC_EXTPARAM 2 + +#define QSV_MAX_ROI_NUM 256 #define QSV_MAX_FRAME_EXT_PARAMS 4 @@ -83,6 +86,7 @@ typedef struct QSVFrame { int num_ext_params; mfxPayload *payloads[QSV_MAX_ENC_PAYLOAD]; ///< used for enc_ctrl.Payload + mfxExtBuffer *extparam[QSV_MAX_ENC_EXTPARAM]; ///< used for enc_ctrl.ExtParam int queued; int used; diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c index fbb22ca436..9abead604b 100644 --- a/libavcodec/qsvenc.c +++ b/libavcodec/qsvenc.c @@ -1501,15 +1501,29 @@ static void free_encoder_ctrl_payloads(mfxEncodeCtrl* enc_ctrl) } } +static void free_encoder_ctrl_extparam(mfxEncodeCtrl* enc_ctrl) +{ + if (enc_ctrl) { + int i; + for (i = 0; i < enc_ctrl->NumExtParam && i < QSV_MAX_ENC_EXTPARAM; i++) { + if (enc_ctrl->ExtParam[i]) + av_freep(&(enc_ctrl->ExtParam[i])); + } + enc_ctrl->NumExtParam = 0; + } +} + static void clear_unused_frames(QSVEncContext *q) { QSVFrame *cur = q->work_frames; while (cur) { if (cur->used && !cur->surface.Data.Locked) { free_encoder_ctrl_payloads(&cur->enc_ctrl); + free_encoder_ctrl_extparam(&cur->enc_ctrl); //do not reuse enc_ctrl from previous frame memset(&cur->enc_ctrl, 0, sizeof(cur->enc_ctrl)); cur->enc_ctrl.Payload = cur->payloads; + cur->enc_ctrl.ExtParam = cur->extparam; if (cur->frame->format == AV_PIX_FMT_QSV) { av_frame_unref(cur->frame); } @@ -1547,6 +1561,7 @@ static int get_free_frame(QSVEncContext *q, QSVFrame **f) return AVERROR(ENOMEM); } frame->enc_ctrl.Payload = frame->payloads; + frame->enc_ctrl.ExtParam = frame->extparam; *last = frame; *f = frame; @@ -1648,6 +1663,70 @@ static void print_interlace_msg(AVCodecContext *avctx, QSVEncContext *q) } } +static int set_roi_encode_ctrl(AVCodecContext *avctx, const AVFrame *frame, + mfxEncodeCtrl *enc_ctrl) +{ +#if QSV_VERSION_ATLEAST(1, 22) + AVFrameSideData *sd = NULL; + int mb_size; + + if (avctx->codec_id == AV_CODEC_ID_H264) { + mb_size = 16; + } else if (avctx->codec_id == AV_CODEC_ID_H265) { + mb_size = 32; + } else { + return 0; + } + + if (frame) + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST); + + if (sd) { + mfxExtEncoderROI *enc_roi = NULL; + AVRegionOfInterest *roi; + uint32_t roi_size; + int nb_roi, i; + + roi = (AVRegionOfInterest *)sd->data; + roi_size = roi->self_size; + if (!roi_size || sd->size % roi_size) { + av_log(avctx, AV_LOG_ERROR, "Invalid ROI Data.\n"); + return AVERROR(EINVAL); + } + nb_roi = sd->size / roi_size; + if (nb_roi > QSV_MAX_ROI_NUM) { + av_log(avctx, AV_LOG_WARNING, "More ROIs set than " + "supported by driver (%d > %d).\n", + nb_roi, QSV_MAX_ROI_NUM); + nb_roi = QSV_MAX_ROI_NUM; + } + + enc_roi = av_mallocz(sizeof(*enc_roi)); + if (!enc_roi) + return AVERROR(ENOMEM); + enc_roi->Header.BufferId = MFX_EXTBUFF_ENCODER_ROI; + enc_roi->Header.BufferSz = sizeof(*enc_roi); + enc_roi->NumROI = nb_roi; + enc_roi->ROIMode = MFX_ROI_MODE_QP_DELTA; + for (i = 0; i < nb_roi; i++) { + roi = (AVRegionOfInterest *)(sd->data + roi_size * i); + enc_roi->ROI[i].Top = FFALIGN(roi->top, mb_size); + enc_roi->ROI[i].Bottom = FFALIGN(roi->bottom, mb_size); + enc_roi->ROI[i].Left = FFALIGN(roi->left, mb_size); + enc_roi->ROI[i].Right = FFALIGN(roi->right, mb_size); + enc_roi->ROI[i].DeltaQP = + roi->qoffset.num * 51 / roi->qoffset.den; + av_log(avctx, AV_LOG_DEBUG, "ROI: (%d,%d)-(%d,%d) -> %+d.\n", + roi->top, roi->left, roi->bottom, roi->right, + enc_roi->ROI[i].DeltaQP); + } + enc_ctrl->ExtParam[enc_ctrl->NumExtParam] = (mfxExtBuffer *)enc_roi; + enc_ctrl->NumExtParam++; + } +#endif + return 0; +} + static int encode_frame(AVCodecContext *avctx, QSVEncContext *q, const AVFrame *frame) { @@ -1714,6 +1793,14 @@ static int encode_frame(AVCodecContext *avctx, QSVEncContext *q, q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl); } + if ((avctx->codec_id == AV_CODEC_ID_H264 || + avctx->codec_id == AV_CODEC_ID_H265) && + enc_ctrl) { + ret = set_roi_encode_ctrl(avctx, frame, enc_ctrl); + if (ret < 0) + goto free; + } + pkt.sync = av_mallocz(sizeof(*pkt.sync)); if (!pkt.sync) goto nomem; @@ -1842,6 +1929,7 @@ int ff_qsv_enc_close(AVCodecContext *avctx, QSVEncContext *q) while (cur) { q->work_frames = cur->next; av_frame_free(&cur->frame); + free_encoder_ctrl_extparam(&cur->enc_ctrl); free_encoder_ctrl_payloads(&cur->enc_ctrl); av_freep(&cur); cur = q->work_frames;
Use the mfxEncoderCtrl parameter to enable ROI. Get side data "AVRegionOfInterest" from filter "addroi" and use it to configure "mfxExtEncoderROI" which is the MediaSDK's ROI configuration. Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> --- libavcodec/qsv_internal.h | 4 ++ libavcodec/qsvenc.c | 88 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+)