Message ID | 20220426100035.2026406-1-wenbin.chen@intel.com |
---|---|
State | New |
Headers | show |
Series | [FFmpeg-devel] libavcodec/qsvdec.c: extract frame packing arrangement data | expand |
Context | Check | Description |
---|---|---|
yinshiyou/make_loongarch64 | success | Make finished |
yinshiyou/make_fate_loongarch64 | success | Make fate finished |
andriy/make_x86 | success | Make finished |
andriy/make_fate_x86 | success | Make fate finished |
andriy/make_armv7_RPi4 | success | Make finished |
andriy/make_fate_armv7_RPi4 | success | Make fate finished |
On Tue, 2022-04-26 at 18:00 +0800, Wenbin Chen wrote: > Use h264_sei to parse SEI data got from MediaSDK. Extract frame > packing arrangement information from SEI data and config AVStereo3D > side data for decoded frame. > > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> > Signed-off-by: Tong Wu <tong1.wu@intel.com> > --- > libavcodec/qsv_internal.h | 2 + > libavcodec/qsvdec.c | 160 ++++++++++++++++++++++++++++++++++++++ > 2 files changed, 162 insertions(+) > > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h > index e2aecdcbd6..a804c392c1 100644 > --- a/libavcodec/qsv_internal.h > +++ b/libavcodec/qsv_internal.h > @@ -54,6 +54,8 @@ > > #define QSV_MAX_FRAME_EXT_PARAMS 4 > > +#define QSV_PAYLOAD_SIZE 1024 > + > #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \ > (MFX_VERSION_MAJOR > (MAJOR) || \ > MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR)) > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c > index 5fc5bed4c8..26fa178b4d 100644 > --- a/libavcodec/qsvdec.c > +++ b/libavcodec/qsvdec.c > @@ -41,13 +41,16 @@ > #include "libavutil/time.h" > #include "libavutil/imgutils.h" > #include "libavutil/film_grain_params.h" > +#include "libavutil/stereo3d.h" > > #include "avcodec.h" > #include "codec_internal.h" > #include "internal.h" > #include "decode.h" > #include "hwconfig.h" > +#include "get_bits.h" > #include "qsv.h" > +#include "h264_sei.h" > #include "qsv_internal.h" > > static const AVRational mfx_tb = { 1, 90000 }; > @@ -101,6 +104,10 @@ typedef struct QSVContext { > > mfxExtBuffer **ext_buffers; > int nb_ext_buffers; > + > + mfxPayload payload; > + H264SEIContext sei; > + H264ParamSets ps; > } QSVContext; > > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { > @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx, > mfxExtAV1FilmGrainParam > } > #endif > > +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame) > +{ > + if (!fpa || !frame) > + return AVERROR(EINVAL); > + > + if (!fpa->arrangement_cancel_flag && > + fpa->arrangement_type <= 6 && > + fpa->content_interpretation_type > 0 && > + fpa->content_interpretation_type < 3) { > + AVStereo3D *stereo = av_stereo3d_create_side_data(frame); > + if (stereo) { > + switch (fpa->arrangement_type) { > + case 0: > + stereo->type = AV_STEREO3D_CHECKERBOARD; > + break; > + case 1: > + stereo->type = AV_STEREO3D_COLUMNS; > + break; > + case 2: > + stereo->type = AV_STEREO3D_LINES; > + break; > + case 3: > + if (fpa->quincunx_sampling_flag) > + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX; > + else > + stereo->type = AV_STEREO3D_SIDEBYSIDE; > + break; > + case 4: > + stereo->type = AV_STEREO3D_TOPBOTTOM; > + break; > + case 5: > + stereo->type = AV_STEREO3D_FRAMESEQUENCE; > + if (fpa->current_frame_is_frame0_flag) > + stereo->view = AV_STEREO3D_VIEW_LEFT; > + else > + stereo->view = AV_STEREO3D_VIEW_RIGHT; > + break; > + case 6: > + stereo->type = AV_STEREO3D_2D; > + break; > + } > + > + if (fpa->content_interpretation_type == 2) > + stereo->flags = AV_STEREO3D_FLAG_INVERT; > + } > + } > + return 0; > +} > + > +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame > *frame) > +{ > + GetBitContext gb_payload; > + uint8_t *sei_buffer; > + int sei_buffer_index; > + int ret; > + > + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT) > + return 0; > + > + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8); > + if (!sei_buffer) { > + av_freep(&sei_buffer); > + return AVERROR(ENOMEM); > + } > + /* remove emulation prevention bytes */ > + sei_buffer_index = 0; > + for (int i = 0; i < q->payload.NumBit / 8; i++) { > + if (q->payload.Data[i] == 3) > + i++; > + sei_buffer[sei_buffer_index] = q->payload.Data[i]; > + sei_buffer_index += 1; > + } > + > + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1); > + if (ret < 0) { > + av_freep(&sei_buffer); > + return ret; > + } > + > + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx); > + if (ret < 0) { > + av_freep(&sei_buffer); > + return ret; > + } > + > + switch (q->payload.Type) { > + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT: > + ret = h264_decode_fpa(&q->sei.frame_packing, frame); > + break; > + default: > + break; > + } > + > + av_freep(&sei_buffer); > + return ret; > +} > + > +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, > AVFrame *frame) > +{ > + mfxU64 ts; > + mfxStatus sts; > + int ret = 0; > + > + if (q->payload.BufSize == 0) { > + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE); > + if (!q->payload.Data) > + return AVERROR(ENOMEM); > + q->payload.BufSize = QSV_PAYLOAD_SIZE; > + } > + > + sts = MFX_ERR_NONE; > + while (sts == MFX_ERR_NONE) { > + > + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload); > + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) { > + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough." > + "Realloc buffer\n"); > + if (q->payload.BufSize >= INT16_MAX / 2) > + return AVERROR(ENOMEM); > + q->payload.BufSize = q->payload.BufSize * 2; > + av_freep(&q->payload.Data); > + q->payload.Data = av_mallocz(q->payload.BufSize); > + if (!q->payload.Data) > + return AVERROR(ENOMEM); > + continue; > + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) { > + break; > + } > + > + switch (avctx->codec_id) { > + case AV_CODEC_ID_H264: > + ret = h264_parse_side_data(avctx, q, frame); > + break; > + default: > + break; > + } > + if (ret < 0) { > + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n"); > + break; > + } > + } > + return ret; > +} > + > static int qsv_decode(AVCodecContext *avctx, QSVContext *q, > AVFrame *frame, int *got_frame, > const AVPacket *avpkt) > @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext > *q, > > outsurf = &aframe.frame->surface; > > + ret = extract_frame_side_data(avctx, q, frame); > + if (ret == AVERROR_INVALIDDATA) > + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n"); > + else if (ret < 0) { > + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from > packet\n"); > + return ret; > + } > + The payload got from the SDK is in encoding order, however the frame returned from MFXVideoDECODE_DecodeFrameAsync() is in display order, see the discussion about MFXVideoDECODE_GetPayload() in https://github.com/Intel-Media-SDK/MediaSDK/issues/2597 . I think it would be better to ask the SDK returns the corresponding info, like what they did in https://github.com/Intel-Media-SDK/MediaSDK/pull/2726 . Thanks Haihao > frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx- > >pkt_timebase); > #if QSV_VERSION_ATLEAST(1, 34) > if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) && > @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q) > av_buffer_unref(&q->frames_ctx.hw_frames_ctx); > av_buffer_unref(&q->frames_ctx.mids_buf); > av_buffer_pool_uninit(&q->pool); > + av_freep(&q->payload.Data); > } > > static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
Wenbin Chen: > Use h264_sei to parse SEI data got from MediaSDK. Extract frame > packing arrangement information from SEI data and config AVStereo3D > side data for decoded frame. > > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> > Signed-off-by: Tong Wu <tong1.wu@intel.com> > --- > libavcodec/qsv_internal.h | 2 + > libavcodec/qsvdec.c | 160 ++++++++++++++++++++++++++++++++++++++ > 2 files changed, 162 insertions(+) > > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h > index e2aecdcbd6..a804c392c1 100644 > --- a/libavcodec/qsv_internal.h > +++ b/libavcodec/qsv_internal.h > @@ -54,6 +54,8 @@ > > #define QSV_MAX_FRAME_EXT_PARAMS 4 > > +#define QSV_PAYLOAD_SIZE 1024 > + > #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \ > (MFX_VERSION_MAJOR > (MAJOR) || \ > MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR)) > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c > index 5fc5bed4c8..26fa178b4d 100644 > --- a/libavcodec/qsvdec.c > +++ b/libavcodec/qsvdec.c > @@ -41,13 +41,16 @@ > #include "libavutil/time.h" > #include "libavutil/imgutils.h" > #include "libavutil/film_grain_params.h" > +#include "libavutil/stereo3d.h" > > #include "avcodec.h" > #include "codec_internal.h" > #include "internal.h" > #include "decode.h" > #include "hwconfig.h" > +#include "get_bits.h" > #include "qsv.h" > +#include "h264_sei.h" > #include "qsv_internal.h" > > static const AVRational mfx_tb = { 1, 90000 }; > @@ -101,6 +104,10 @@ typedef struct QSVContext { > > mfxExtBuffer **ext_buffers; > int nb_ext_buffers; > + > + mfxPayload payload; > + H264SEIContext sei; > + H264ParamSets ps; > } QSVContext; > > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { > @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam > } > #endif > > +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame) > +{ > + if (!fpa || !frame) > + return AVERROR(EINVAL); > + > + if (!fpa->arrangement_cancel_flag && > + fpa->arrangement_type <= 6 && > + fpa->content_interpretation_type > 0 && > + fpa->content_interpretation_type < 3) { > + AVStereo3D *stereo = av_stereo3d_create_side_data(frame); > + if (stereo) { > + switch (fpa->arrangement_type) { > + case 0: > + stereo->type = AV_STEREO3D_CHECKERBOARD; > + break; > + case 1: > + stereo->type = AV_STEREO3D_COLUMNS; > + break; > + case 2: > + stereo->type = AV_STEREO3D_LINES; > + break; > + case 3: > + if (fpa->quincunx_sampling_flag) > + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX; > + else > + stereo->type = AV_STEREO3D_SIDEBYSIDE; > + break; > + case 4: > + stereo->type = AV_STEREO3D_TOPBOTTOM; > + break; > + case 5: > + stereo->type = AV_STEREO3D_FRAMESEQUENCE; > + if (fpa->current_frame_is_frame0_flag) > + stereo->view = AV_STEREO3D_VIEW_LEFT; > + else > + stereo->view = AV_STEREO3D_VIEW_RIGHT; > + break; > + case 6: > + stereo->type = AV_STEREO3D_2D; > + break; > + } > + > + if (fpa->content_interpretation_type == 2) > + stereo->flags = AV_STEREO3D_FLAG_INVERT; > + } > + } > + return 0; > +} > + > +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) > +{ > + GetBitContext gb_payload; > + uint8_t *sei_buffer; > + int sei_buffer_index; > + int ret; > + > + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT) > + return 0; > + > + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8); > + if (!sei_buffer) { > + av_freep(&sei_buffer); > + return AVERROR(ENOMEM); > + } > + /* remove emulation prevention bytes */ > + sei_buffer_index = 0; > + for (int i = 0; i < q->payload.NumBit / 8; i++) { > + if (q->payload.Data[i] == 3) > + i++; > + sei_buffer[sei_buffer_index] = q->payload.Data[i]; > + sei_buffer_index += 1; Not every 0x03 is an emulation prevention byte. > + } > + > + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1); Buffers used with the get-bits API need to be padded. Best to use av_fast_padded_malloc and ff_h2645_extract_rbsp. > + if (ret < 0) { > + av_freep(&sei_buffer); > + return ret; > + } > + > + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx); > + if (ret < 0) { > + av_freep(&sei_buffer); > + return ret; > + } > + > + switch (q->payload.Type) { > + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT: > + ret = h264_decode_fpa(&q->sei.frame_packing, frame); > + break; > + default: > + break; > + } > + > + av_freep(&sei_buffer); > + return ret; > +} > + > +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) > +{ > + mfxU64 ts; > + mfxStatus sts; > + int ret = 0; > + > + if (q->payload.BufSize == 0) { > + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE); > + if (!q->payload.Data) > + return AVERROR(ENOMEM); > + q->payload.BufSize = QSV_PAYLOAD_SIZE; > + } > + > + sts = MFX_ERR_NONE; > + while (sts == MFX_ERR_NONE) { > + > + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload); > + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) { > + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough." > + "Realloc buffer\n"); Why this warning? Why a log-message at all? > + if (q->payload.BufSize >= INT16_MAX / 2) Where does this limit come from? > + return AVERROR(ENOMEM); > + q->payload.BufSize = q->payload.BufSize * 2; > + av_freep(&q->payload.Data); > + q->payload.Data = av_mallocz(q->payload.BufSize); > + if (!q->payload.Data) > + return AVERROR(ENOMEM); q->payload.Data is NULL if this happens, but q->payload.BufSize will be > 0, so that q->payload used in the next MFXVideoDECODE_GetPayload call will be inconsistent. > + continue; > + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) { > + break; > + } > + > + switch (avctx->codec_id) { > + case AV_CODEC_ID_H264: > + ret = h264_parse_side_data(avctx, q, frame); > + break; > + default: > + break; > + } > + if (ret < 0) { > + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n"); > + break; > + } > + } > + return ret; > +} > + > static int qsv_decode(AVCodecContext *avctx, QSVContext *q, > AVFrame *frame, int *got_frame, > const AVPacket *avpkt) > @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q, > > outsurf = &aframe.frame->surface; > > + ret = extract_frame_side_data(avctx, q, frame); > + if (ret == AVERROR_INVALIDDATA) > + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n"); > + else if (ret < 0) { > + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from packet\n"); > + return ret; > + } > + > frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase); > #if QSV_VERSION_ATLEAST(1, 34) > if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) && > @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q) > av_buffer_unref(&q->frames_ctx.hw_frames_ctx); > av_buffer_unref(&q->frames_ctx.mids_buf); > av_buffer_pool_uninit(&q->pool); > + av_freep(&q->payload.Data); > } > > static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
> On Tue, 2022-04-26 at 18:00 +0800, Wenbin Chen wrote: > > Use h264_sei to parse SEI data got from MediaSDK. Extract frame > > packing arrangement information from SEI data and config AVStereo3D > > side data for decoded frame. > > > > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com> > > Signed-off-by: Tong Wu <tong1.wu@intel.com> > > --- > > libavcodec/qsv_internal.h | 2 + > > libavcodec/qsvdec.c | 160 > ++++++++++++++++++++++++++++++++++++++ > > 2 files changed, 162 insertions(+) > > > > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h > > index e2aecdcbd6..a804c392c1 100644 > > --- a/libavcodec/qsv_internal.h > > +++ b/libavcodec/qsv_internal.h > > @@ -54,6 +54,8 @@ > > > > #define QSV_MAX_FRAME_EXT_PARAMS 4 > > > > +#define QSV_PAYLOAD_SIZE 1024 > > + > > #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \ > > (MFX_VERSION_MAJOR > (MAJOR) || \ > > MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= > (MINOR)) > > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c > > index 5fc5bed4c8..26fa178b4d 100644 > > --- a/libavcodec/qsvdec.c > > +++ b/libavcodec/qsvdec.c > > @@ -41,13 +41,16 @@ > > #include "libavutil/time.h" > > #include "libavutil/imgutils.h" > > #include "libavutil/film_grain_params.h" > > +#include "libavutil/stereo3d.h" > > > > #include "avcodec.h" > > #include "codec_internal.h" > > #include "internal.h" > > #include "decode.h" > > #include "hwconfig.h" > > +#include "get_bits.h" > > #include "qsv.h" > > +#include "h264_sei.h" > > #include "qsv_internal.h" > > > > static const AVRational mfx_tb = { 1, 90000 }; > > @@ -101,6 +104,10 @@ typedef struct QSVContext { > > > > mfxExtBuffer **ext_buffers; > > int nb_ext_buffers; > > + > > + mfxPayload payload; > > + H264SEIContext sei; > > + H264ParamSets ps; > > } QSVContext; > > > > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { > > @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext > *avctx, > > mfxExtAV1FilmGrainParam > > } > > #endif > > > > +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame) > > +{ > > + if (!fpa || !frame) > > + return AVERROR(EINVAL); > > + > > + if (!fpa->arrangement_cancel_flag && > > + fpa->arrangement_type <= 6 && > > + fpa->content_interpretation_type > 0 && > > + fpa->content_interpretation_type < 3) { > > + AVStereo3D *stereo = av_stereo3d_create_side_data(frame); > > + if (stereo) { > > + switch (fpa->arrangement_type) { > > + case 0: > > + stereo->type = AV_STEREO3D_CHECKERBOARD; > > + break; > > + case 1: > > + stereo->type = AV_STEREO3D_COLUMNS; > > + break; > > + case 2: > > + stereo->type = AV_STEREO3D_LINES; > > + break; > > + case 3: > > + if (fpa->quincunx_sampling_flag) > > + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX; > > + else > > + stereo->type = AV_STEREO3D_SIDEBYSIDE; > > + break; > > + case 4: > > + stereo->type = AV_STEREO3D_TOPBOTTOM; > > + break; > > + case 5: > > + stereo->type = AV_STEREO3D_FRAMESEQUENCE; > > + if (fpa->current_frame_is_frame0_flag) > > + stereo->view = AV_STEREO3D_VIEW_LEFT; > > + else > > + stereo->view = AV_STEREO3D_VIEW_RIGHT; > > + break; > > + case 6: > > + stereo->type = AV_STEREO3D_2D; > > + break; > > + } > > + > > + if (fpa->content_interpretation_type == 2) > > + stereo->flags = AV_STEREO3D_FLAG_INVERT; > > + } > > + } > > + return 0; > > +} > > + > > +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, > AVFrame > > *frame) > > +{ > > + GetBitContext gb_payload; > > + uint8_t *sei_buffer; > > + int sei_buffer_index; > > + int ret; > > + > > + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT) > > + return 0; > > + > > + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8); > > + if (!sei_buffer) { > > + av_freep(&sei_buffer); > > + return AVERROR(ENOMEM); > > + } > > + /* remove emulation prevention bytes */ > > + sei_buffer_index = 0; > > + for (int i = 0; i < q->payload.NumBit / 8; i++) { > > + if (q->payload.Data[i] == 3) > > + i++; > > + sei_buffer[sei_buffer_index] = q->payload.Data[i]; > > + sei_buffer_index += 1; > > + } > > + > > + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1); > > + if (ret < 0) { > > + av_freep(&sei_buffer); > > + return ret; > > + } > > + > > + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx); > > + if (ret < 0) { > > + av_freep(&sei_buffer); > > + return ret; > > + } > > + > > + switch (q->payload.Type) { > > + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT: > > + ret = h264_decode_fpa(&q->sei.frame_packing, frame); > > + break; > > + default: > > + break; > > + } > > + > > + av_freep(&sei_buffer); > > + return ret; > > +} > > + > > +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext > *q, > > AVFrame *frame) > > +{ > > + mfxU64 ts; > > + mfxStatus sts; > > + int ret = 0; > > + > > + if (q->payload.BufSize == 0) { > > + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE); > > + if (!q->payload.Data) > > + return AVERROR(ENOMEM); > > + q->payload.BufSize = QSV_PAYLOAD_SIZE; > > + } > > + > > + sts = MFX_ERR_NONE; > > + while (sts == MFX_ERR_NONE) { > > + > > + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload); > > + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) { > > + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough." > > + "Realloc buffer\n"); > > + if (q->payload.BufSize >= INT16_MAX / 2) > > + return AVERROR(ENOMEM); > > + q->payload.BufSize = q->payload.BufSize * 2; > > + av_freep(&q->payload.Data); > > + q->payload.Data = av_mallocz(q->payload.BufSize); > > + if (!q->payload.Data) > > + return AVERROR(ENOMEM); > > + continue; > > + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) { > > + break; > > + } > > + > > + switch (avctx->codec_id) { > > + case AV_CODEC_ID_H264: > > + ret = h264_parse_side_data(avctx, q, frame); > > + break; > > + default: > > + break; > > + } > > + if (ret < 0) { > > + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n"); > > + break; > > + } > > + } > > + return ret; > > +} > > + > > static int qsv_decode(AVCodecContext *avctx, QSVContext *q, > > AVFrame *frame, int *got_frame, > > const AVPacket *avpkt) > > @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, > QSVContext > > *q, > > > > outsurf = &aframe.frame->surface; > > > > + ret = extract_frame_side_data(avctx, q, frame); > > + if (ret == AVERROR_INVALIDDATA) > > + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n"); > > + else if (ret < 0) { > > + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from > > packet\n"); > > + return ret; > > + } > > + > > The payload got from the SDK is in encoding order, however the frame > returned > from MFXVideoDECODE_DecodeFrameAsync() is in display order, see the > discussion > about MFXVideoDECODE_GetPayload() in > https://github.com/Intel-Media-SDK/MediaSDK/issues/2597 . I think it would > be > better to ask the SDK returns the corresponding info, like what they did in > https://github.com/Intel-Media-SDK/MediaSDK/pull/2726 . > > Thanks > Haihao Thanks for your information. I will try to fix this problem. Thanks Wenbin > > > > > frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx- > > >pkt_timebase); > > #if QSV_VERSION_ATLEAST(1, 34) > > if ((avctx->export_side_data & > AV_CODEC_EXPORT_DATA_FILM_GRAIN) && > > @@ -770,6 +929,7 @@ static void > qsv_decode_close_qsvcontext(QSVContext *q) > > av_buffer_unref(&q->frames_ctx.hw_frames_ctx); > > av_buffer_unref(&q->frames_ctx.mids_buf); > > av_buffer_pool_uninit(&q->pool); > > + av_freep(&q->payload.Data); > > } > > > > static int qsv_process_data(AVCodecContext *avctx, QSVContext *q, > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h index e2aecdcbd6..a804c392c1 100644 --- a/libavcodec/qsv_internal.h +++ b/libavcodec/qsv_internal.h @@ -54,6 +54,8 @@ #define QSV_MAX_FRAME_EXT_PARAMS 4 +#define QSV_PAYLOAD_SIZE 1024 + #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \ (MFX_VERSION_MAJOR > (MAJOR) || \ MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR)) diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c index 5fc5bed4c8..26fa178b4d 100644 --- a/libavcodec/qsvdec.c +++ b/libavcodec/qsvdec.c @@ -41,13 +41,16 @@ #include "libavutil/time.h" #include "libavutil/imgutils.h" #include "libavutil/film_grain_params.h" +#include "libavutil/stereo3d.h" #include "avcodec.h" #include "codec_internal.h" #include "internal.h" #include "decode.h" #include "hwconfig.h" +#include "get_bits.h" #include "qsv.h" +#include "h264_sei.h" #include "qsv_internal.h" static const AVRational mfx_tb = { 1, 90000 }; @@ -101,6 +104,10 @@ typedef struct QSVContext { mfxExtBuffer **ext_buffers; int nb_ext_buffers; + + mfxPayload payload; + H264SEIContext sei; + H264ParamSets ps; } QSVContext; static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam } #endif +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame) +{ + if (!fpa || !frame) + return AVERROR(EINVAL); + + if (!fpa->arrangement_cancel_flag && + fpa->arrangement_type <= 6 && + fpa->content_interpretation_type > 0 && + fpa->content_interpretation_type < 3) { + AVStereo3D *stereo = av_stereo3d_create_side_data(frame); + if (stereo) { + switch (fpa->arrangement_type) { + case 0: + stereo->type = AV_STEREO3D_CHECKERBOARD; + break; + case 1: + stereo->type = AV_STEREO3D_COLUMNS; + break; + case 2: + stereo->type = AV_STEREO3D_LINES; + break; + case 3: + if (fpa->quincunx_sampling_flag) + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX; + else + stereo->type = AV_STEREO3D_SIDEBYSIDE; + break; + case 4: + stereo->type = AV_STEREO3D_TOPBOTTOM; + break; + case 5: + stereo->type = AV_STEREO3D_FRAMESEQUENCE; + if (fpa->current_frame_is_frame0_flag) + stereo->view = AV_STEREO3D_VIEW_LEFT; + else + stereo->view = AV_STEREO3D_VIEW_RIGHT; + break; + case 6: + stereo->type = AV_STEREO3D_2D; + break; + } + + if (fpa->content_interpretation_type == 2) + stereo->flags = AV_STEREO3D_FLAG_INVERT; + } + } + return 0; +} + +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) +{ + GetBitContext gb_payload; + uint8_t *sei_buffer; + int sei_buffer_index; + int ret; + + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT) + return 0; + + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8); + if (!sei_buffer) { + av_freep(&sei_buffer); + return AVERROR(ENOMEM); + } + /* remove emulation prevention bytes */ + sei_buffer_index = 0; + for (int i = 0; i < q->payload.NumBit / 8; i++) { + if (q->payload.Data[i] == 3) + i++; + sei_buffer[sei_buffer_index] = q->payload.Data[i]; + sei_buffer_index += 1; + } + + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1); + if (ret < 0) { + av_freep(&sei_buffer); + return ret; + } + + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx); + if (ret < 0) { + av_freep(&sei_buffer); + return ret; + } + + switch (q->payload.Type) { + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT: + ret = h264_decode_fpa(&q->sei.frame_packing, frame); + break; + default: + break; + } + + av_freep(&sei_buffer); + return ret; +} + +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) +{ + mfxU64 ts; + mfxStatus sts; + int ret = 0; + + if (q->payload.BufSize == 0) { + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE); + if (!q->payload.Data) + return AVERROR(ENOMEM); + q->payload.BufSize = QSV_PAYLOAD_SIZE; + } + + sts = MFX_ERR_NONE; + while (sts == MFX_ERR_NONE) { + + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload); + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough." + "Realloc buffer\n"); + if (q->payload.BufSize >= INT16_MAX / 2) + return AVERROR(ENOMEM); + q->payload.BufSize = q->payload.BufSize * 2; + av_freep(&q->payload.Data); + q->payload.Data = av_mallocz(q->payload.BufSize); + if (!q->payload.Data) + return AVERROR(ENOMEM); + continue; + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) { + break; + } + + switch (avctx->codec_id) { + case AV_CODEC_ID_H264: + ret = h264_parse_side_data(avctx, q, frame); + break; + default: + break; + } + if (ret < 0) { + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n"); + break; + } + } + return ret; +} + static int qsv_decode(AVCodecContext *avctx, QSVContext *q, AVFrame *frame, int *got_frame, const AVPacket *avpkt) @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q, outsurf = &aframe.frame->surface; + ret = extract_frame_side_data(avctx, q, frame); + if (ret == AVERROR_INVALIDDATA) + av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n"); + else if (ret < 0) { + av_log(avctx, AV_LOG_ERROR, "Error extracting side data from packet\n"); + return ret; + } + frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase); #if QSV_VERSION_ATLEAST(1, 34) if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) && @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q) av_buffer_unref(&q->frames_ctx.hw_frames_ctx); av_buffer_unref(&q->frames_ctx.mids_buf); av_buffer_pool_uninit(&q->pool); + av_freep(&q->payload.Data); } static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,