From patchwork Tue Jan 26 02:32:07 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Wenbin Chen X-Patchwork-Id: 25184 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 78D6144BD9F for ; Tue, 26 Jan 2021 04:36:13 +0200 (EET) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 5ED0068A0F0; Tue, 26 Jan 2021 04:36:13 +0200 (EET) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga12.intel.com (mga12.intel.com [192.55.52.136]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 7091F689E13 for ; Tue, 26 Jan 2021 04:36:05 +0200 (EET) IronPort-SDR: Vd90ljdemXRrNGIE82hIJ/zaWg0CMFaMi5hHgupRu3YcZDH6y4mwqbirV/sMfrJdu1xKMb2XGW JkoZggjMvG+g== X-IronPort-AV: E=McAfee;i="6000,8403,9875"; a="159015222" X-IronPort-AV: E=Sophos;i="5.79,375,1602572400"; d="scan'208";a="159015222" Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by fmsmga106.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 25 Jan 2021 18:36:04 -0800 IronPort-SDR: NoX/aS8jqlZBPLX4sFyI9+ehScLte7e8ffu/iCNBer/eFoLDhuSulfsBEnqZ0kKEfW70K0FN1M Px2Qq5nMSIMQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.79,375,1602572400"; d="scan'208";a="361797067" Received: from chenwenbin-z390-aorus-ultra.sh.intel.com ([10.239.35.3]) by fmsmga008.fm.intel.com with ESMTP; 25 Jan 2021 18:36:02 -0800 From: wenbin.chen@intel.com To: ffmpeg-devel@ffmpeg.org Date: Tue, 26 Jan 2021 10:32:07 +0800 Message-Id: <20210126023213.146185-4-wenbin.chen@intel.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20210126023213.146185-1-wenbin.chen@intel.com> References: <20210126023213.146185-1-wenbin.chen@intel.com> MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH 04/10] libavcodec/qsvdec.c: extract frame packing arrangement data X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Wenbinc-Bin Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" From: Wenbinc-Bin Use h264_sei to parse SEI data got from MediaSDK. Extract frame packing arrangement information from SEI data and config AVStereo3D side data for decoded frame. Sigend-off-by: Wenbin Chen --- libavcodec/qsv_internal.h | 2 + libavcodec/qsvdec.c | 149 ++++++++++++++++++++++++++++++++++++++ libavcodec/qsvdec.h | 6 ++ 3 files changed, 157 insertions(+) diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h index 6b2fbbe252..aa3da97c99 100644 --- a/libavcodec/qsv_internal.h +++ b/libavcodec/qsv_internal.h @@ -52,6 +52,8 @@ #define QSV_MAX_ENC_PAYLOAD 2 // # of mfxEncodeCtrl payloads supported +#define QSV_PAYLOAD_SIZE 1024 + #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \ (MFX_VERSION_MAJOR > (MAJOR) || \ MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR)) diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c index d10f90a0db..5cb49cfb4a 100644 --- a/libavcodec/qsvdec.c +++ b/libavcodec/qsvdec.c @@ -36,10 +36,12 @@ #include "libavutil/pixfmt.h" #include "libavutil/time.h" #include "libavutil/imgutils.h" +#include "libavutil/stereo3d.h" #include "avcodec.h" #include "internal.h" #include "decode.h" +#include "get_bits.h" #include "qsv.h" #include "qsv_internal.h" #include "qsvdec.h" @@ -398,6 +400,147 @@ static QSVFrame *find_frame(QSVContext *q, mfxFrameSurface1 *surf) return NULL; } +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame) +{ + if (!fpa || !frame) { + return AVERROR(EINVAL); + } + + if (!fpa->arrangement_cancel_flag && + fpa->arrangement_type <= 6 && + fpa->content_interpretation_type > 0 && + fpa->content_interpretation_type < 3) { + AVStereo3D *stereo = av_stereo3d_create_side_data(frame); + if (stereo) { + switch (fpa->arrangement_type) { + case 0: + stereo->type = AV_STEREO3D_CHECKERBOARD; + break; + case 1: + stereo->type = AV_STEREO3D_COLUMNS; + break; + case 2: + stereo->type = AV_STEREO3D_LINES; + break; + case 3: + if (fpa->quincunx_sampling_flag) + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX; + else + stereo->type = AV_STEREO3D_SIDEBYSIDE; + break; + case 4: + stereo->type = AV_STEREO3D_TOPBOTTOM; + break; + case 5: + stereo->type = AV_STEREO3D_FRAMESEQUENCE; + if (fpa->current_frame_is_frame0_flag) + stereo->view = AV_STEREO3D_VIEW_LEFT; + else + stereo->view = AV_STEREO3D_VIEW_RIGHT; + break; + case 6: + stereo->type = AV_STEREO3D_2D; + break; + } + + if (fpa->content_interpretation_type == 2) + stereo->flags = AV_STEREO3D_FLAG_INVERT; + } + } + return 0; +} + +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) +{ + GetBitContext gb_payload; + uint8_t *sei_buffer; + int sei_buffer_index; + int ret; + + /* remove emulation prevention bytes */ + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8); + if (!sei_buffer) { + av_freep(&sei_buffer); + return AVERROR(ENOMEM); + } + sei_buffer_index = 0; + for (int i = 0; i < q->payload.NumBit / 8; i++) { + if (q->payload.Data[i] == 3) + i++; + sei_buffer[sei_buffer_index] = q->payload.Data[i]; + sei_buffer_index += 1; + } + + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1); + if (ret < 0) { + av_freep(&sei_buffer); + return ret; + } + + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx); + if (ret < 0) { + av_freep(&sei_buffer); + return ret; + } + + switch (q->payload.Type) { + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT: + ret = h264_decode_fpa(&q->sei.frame_packing, frame); + break; + default: + break; + } + + av_freep(&sei_buffer); + return ret; +} + +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) +{ + mfxU64 ts; + mfxStatus sts; + int ret; + + if (q->payload.BufSize == 0) { + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE); + if (!q->payload.Data) { + av_freep(&q->payload.Data); + return AVERROR(ENOMEM); + } + q->payload.BufSize = QSV_PAYLOAD_SIZE; + } + + sts = MFX_ERR_NONE; + while (sts == MFX_ERR_NONE) { + + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload); + + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough. One SEI will be skipped\n"); + continue; + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) { + break; + } + + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT) + continue; + + switch (avctx->codec_id) { + case AV_CODEC_ID_H264: + ret = h264_parse_side_data(avctx, q, frame); + break; + default: + break; + } + + if (ret < 0) { + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n"); + break; + } + } + return ret; +} + static int qsv_decode(AVCodecContext *avctx, QSVContext *q, AVFrame *frame, int *got_frame, AVPacket *avpkt) @@ -501,6 +644,10 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q, outsurf = &out_frame->surface; + ret = extract_frame_side_data(avctx, q, frame); + if (ret < 0) + av_log(avctx, AV_LOG_WARNING, "Extracting side from packet failed\n"); + #if FF_API_PKT_PTS FF_DISABLE_DEPRECATION_WARNINGS frame->pkt_pts = outsurf->Data.TimeStamp; @@ -564,6 +711,8 @@ int ff_qsv_decode_close(QSVContext *q) av_buffer_unref(&q->frames_ctx.mids_buf); av_buffer_pool_uninit(&q->pool); + av_freep(&q->payload.Data); + return 0; } diff --git a/libavcodec/qsvdec.h b/libavcodec/qsvdec.h index f3b7344cba..282943b00c 100644 --- a/libavcodec/qsvdec.h +++ b/libavcodec/qsvdec.h @@ -34,6 +34,7 @@ #include "avcodec.h" #include "hwconfig.h" +#include "h264_sei.h" #include "qsv_internal.h" typedef struct QSVContext { @@ -70,8 +71,13 @@ typedef struct QSVContext { char *load_plugins; + mfxPayload payload; + mfxExtBuffer **ext_buffers; int nb_ext_buffers; + + H264SEIContext sei; + H264ParamSets ps; } QSVContext; extern const AVCodecHWConfigInternal *const ff_qsv_hw_configs[];