From patchwork Wed Dec 25 11:02:15 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 16941 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 9F05744A33D for ; Wed, 25 Dec 2019 13:10:12 +0200 (EET) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 2090168A435; Wed, 25 Dec 2019 13:10:12 +0200 (EET) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga18.intel.com (mga18.intel.com [134.134.136.126]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id CCA3968A0A1 for ; Wed, 25 Dec 2019 13:10:04 +0200 (EET) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga002.jf.intel.com ([10.7.209.21]) by orsmga106.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 25 Dec 2019 03:10:01 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,354,1571727600"; d="scan'208";a="229871789" Received: from yguo18-skl-u1604.sh.intel.com ([10.239.13.25]) by orsmga002.jf.intel.com with ESMTP; 25 Dec 2019 03:10:00 -0800 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Wed, 25 Dec 2019 19:02:15 +0800 Message-Id: <1577271735-9180-1-git-send-email-yejun.guo@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [FFmpeg-devel] [PATCH] qsvenc: Add ROI encoding support for h264_qsv and hevc_qsv X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" command examples: ./ffmpeg -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:0.5 -c:v hevc_qsv -b:v 5M -y qsv.b5m.roi.h265 ./ffmpeg -s 1920x1080 -i input.yuv -vf addroi=0:0:640:320:-0.5 -c:v h264_qsv -mfmode 1 -b:v 4M -y qsv.b4m.roi.h264 Signed-off-by: Guo, Yejun --- libavcodec/qsvenc.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++ libavcodec/qsvenc.h | 12 +++++- 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c index 9e41650..aa42c9b 100644 --- a/libavcodec/qsvenc.c +++ b/libavcodec/qsvenc.c @@ -482,6 +482,7 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) int target_bitrate_kbps, max_bitrate_kbps, brc_param_multiplier; int buffer_size_in_kilobytes, initial_delay_in_kilobytes; int ret; + av_unused int mfe_enabled = 0; ret = ff_qsv_codec_id_to_mfx(avctx->codec_id); if (ret < 0) @@ -755,6 +756,7 @@ FF_ENABLE_DEPRECATION_WARNINGS q->extmfp.Header.BufferSz = sizeof(q->extmfp); q->extmfp.MFMode = q->mfmode; + mfe_enabled = q->mfmode != MFX_MF_DISABLED; av_log(avctx,AV_LOG_VERBOSE,"MFMode:%d\n", q->extmfp.MFMode); q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extmfp; } @@ -790,6 +792,25 @@ FF_ENABLE_DEPRECATION_WARNINGS } #endif +#if QSV_HAVE_ROI_ENCODING + // mfe and roi could not be enabled together, they even could not be queried together, + // otherwise, gpu hang. + if (!mfe_enabled) { + q->extroi.Header.BufferId = MFX_EXTBUFF_ENCODER_ROI; + q->extroi.Header.BufferSz = sizeof(q->extroi); + q->extroi.ROIMode = MFX_ROI_MODE_QP_DELTA; + // 256 to query the maximum supported value + q->extroi.NumROI = 256; + // due to the requirement of msdk, we must set non_empty rect for query + for (int i = 0; i < sizeof(q->extroi.ROI)/sizeof(q->extroi.ROI[0]); ++i) { + q->extroi.ROI[i].Right = 16; + q->extroi.ROI[i].Bottom = 16; + } + q->roi_index_in_internal_buffer = q->nb_extparam_internal; + q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extroi; + } +#endif + if (!check_enc_param(avctx,q)) { av_log(avctx, AV_LOG_ERROR, "some encoding parameters are not supported by the QSV " @@ -1178,6 +1199,10 @@ int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q) "Error querying encoder params"); } +#if QSV_HAVE_ROI_ENCODING + q->roi_max_regions = q->extroi.NumROI; +#endif + ret = MFXVideoENCODE_QueryIOSurf(q->session, &q->param, &q->req); if (ret < 0) return ff_qsv_print_error(avctx, ret, @@ -1375,6 +1400,88 @@ static void print_interlace_msg(AVCodecContext *avctx, QSVEncContext *q) } } +static int qsv_encode_set_roi(AVCodecContext *avctx, QSVEncContext *q, const AVFrame *frame, mfxEncodeCtrl *enc_ctrl) +{ +#if QSV_HAVE_ROI_ENCODING + // in qsv, the min_delta_qp is -51, and the max_delta_qp is 51 + int qp_range = 51; + int nb_roi; + const AVRegionOfInterest *roi; + uint32_t roi_size; + + AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST); + if (!sd) { + // need to reset for the case: with roi side data --> no roi side data + enc_ctrl->NumExtParam = 0; + return 0; + } + +#if QSV_HAVE_MF + if (avctx->codec_id == AV_CODEC_ID_H264) { + if (q->mfmode != MFX_MF_DISABLED) { + if (!q->roi_warned) { + q->roi_warned = 1; + av_log(avctx, AV_LOG_WARNING, "to enable ROI, " + "please make sure Multi-Frame Mode is disabled.\n"); + } + return 0; + } + } +#endif + + // msdk currently just supports roi encoding for h264 and h265 + if (avctx->codec_id != AV_CODEC_ID_H264 && avctx->codec_id != AV_CODEC_ID_HEVC) { + if (!q->roi_warned) { + q->roi_warned = 1; + av_log(avctx, AV_LOG_WARNING, "ROI encoding is only enabled for h264_qsv and hevc_qsv.\n"); + } + return 0; + } + + roi = (const AVRegionOfInterest*)sd->data; + roi_size = roi->self_size; + if (!roi_size || sd->size % roi_size != 0) { + av_log(avctx, AV_LOG_ERROR, "Invalid AVRegionOfInterest.self_size.\n"); + return AVERROR(EINVAL); + } + + nb_roi = sd->size / roi_size; + if (nb_roi > q->roi_max_regions) { + if (!q->roi_warned) { + av_log(avctx, AV_LOG_WARNING, "More ROIs set than supported (%d > %d).\n", + nb_roi, q->roi_max_regions); + q->roi_warned = 1; + } + nb_roi = q->roi_max_regions; + } + q->extroi.NumROI = nb_roi; + + // For overlapping regions, the first in the array takes priority. + for (int i = 0; i < nb_roi; i++) { + float qoffset; + + roi = (const AVRegionOfInterest*)(sd->data + roi_size * i); + if (roi->qoffset.den == 0) { + av_log(avctx, AV_LOG_ERROR, "AVRegionOfInterest.qoffset.den must not be zero.\n"); + return AVERROR(EINVAL); + } + qoffset = roi->qoffset.num * 1.0f / roi->qoffset.den; + qoffset = av_clipf(qoffset * qp_range, -qp_range, +qp_range); + + q->extroi.ROI[i].Left = roi->left; + q->extroi.ROI[i].Right = roi->right; + q->extroi.ROI[i].Top = roi->top; + q->extroi.ROI[i].Bottom = roi->bottom; + q->extroi.ROI[i].DeltaQP = qoffset; + } + + enc_ctrl->NumExtParam = 1; + enc_ctrl->ExtParam = &q->extparam_internal[q->roi_index_in_internal_buffer]; +#endif + + return 0; +} + static int encode_frame(AVCodecContext *avctx, QSVEncContext *q, const AVFrame *frame) { @@ -1445,6 +1552,12 @@ static int encode_frame(AVCodecContext *avctx, QSVEncContext *q, q->set_encode_ctrl_cb(avctx, frame, &qsv_frame->enc_ctrl); } + if (frame) { + ret = qsv_encode_set_roi(avctx, q, frame, &qsv_frame->enc_ctrl); + if (ret < 0) + return ret; + } + sync = av_mallocz(sizeof(*sync)); if (!sync) { av_freep(&bs); diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h index 6609171..9c6caf1 100644 --- a/libavcodec/qsvenc.h +++ b/libavcodec/qsvenc.h @@ -40,6 +40,7 @@ #define QSV_HAVE_EXT_HEVC_TILES QSV_VERSION_ATLEAST(1, 13) #define QSV_HAVE_EXT_VP9_PARAM QSV_VERSION_ATLEAST(1, 26) +#define QSV_HAVE_ROI_ENCODING QSV_VERSION_ATLEAST(1, 22) #define QSV_HAVE_TRELLIS QSV_VERSION_ATLEAST(1, 8) #define QSV_HAVE_MAX_SLICE_SIZE QSV_VERSION_ATLEAST(1, 9) @@ -131,12 +132,21 @@ typedef struct QSVEncContext { #if QSV_HAVE_EXT_VP9_PARAM mfxExtVP9Param extvp9param; #endif +#if QSV_HAVE_ROI_ENCODING + mfxExtEncoderROI extroi; + // Maximum number of regions supported + int roi_max_regions; + int roi_index_in_internal_buffer; + // If the driver does not support ROI then warn the first time we + // encounter a frame with ROI side data. + int roi_warned; +#endif mfxExtOpaqueSurfaceAlloc opaque_alloc; mfxFrameSurface1 **opaque_surfaces; AVBufferRef *opaque_alloc_buf; - mfxExtBuffer *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2)]; + mfxExtBuffer *extparam_internal[2 + QSV_HAVE_CO2 + QSV_HAVE_CO3 + (QSV_HAVE_MF * 2) + QSV_HAVE_ROI_ENCODING]; int nb_extparam_internal; mfxExtBuffer **extparam;