diff mbox series

[FFmpeg-devel] libavcodec/qsvdec.c: extract frame packing arrangement data

Message ID 20220426100035.2026406-1-wenbin.chen@intel.com
State New
Headers show
Series [FFmpeg-devel] libavcodec/qsvdec.c: extract frame packing arrangement data | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished
andriy/make_armv7_RPi4 success Make finished
andriy/make_fate_armv7_RPi4 success Make fate finished

Commit Message

Chen, Wenbin April 26, 2022, 10 a.m. UTC 
Use h264_sei to parse SEI data got from MediaSDK. Extract frame
packing arrangement information from SEI data and config AVStereo3D
side data for decoded frame.

Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
Signed-off-by: Tong Wu <tong1.wu@intel.com>
---
 libavcodec/qsv_internal.h |   2 +
 libavcodec/qsvdec.c       | 160 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 162 insertions(+)

Comments

Xiang, Haihao April 27, 2022, 9:41 a.m. UTC | #1 
On Tue, 2022-04-26 at 18:00 +0800, Wenbin Chen wrote:
> Use h264_sei to parse SEI data got from MediaSDK. Extract frame
> packing arrangement information from SEI data and config AVStereo3D
> side data for decoded frame.
> 
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> Signed-off-by: Tong Wu <tong1.wu@intel.com>
> ---
>  libavcodec/qsv_internal.h |   2 +
>  libavcodec/qsvdec.c       | 160 ++++++++++++++++++++++++++++++++++++++
>  2 files changed, 162 insertions(+)
> 
> diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> index e2aecdcbd6..a804c392c1 100644
> --- a/libavcodec/qsv_internal.h
> +++ b/libavcodec/qsv_internal.h
> @@ -54,6 +54,8 @@
>  
>  #define QSV_MAX_FRAME_EXT_PARAMS 4
>  
> +#define QSV_PAYLOAD_SIZE 1024
> +
>  #define QSV_VERSION_ATLEAST(MAJOR, MINOR)   \
>      (MFX_VERSION_MAJOR > (MAJOR) ||         \
>       MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 5fc5bed4c8..26fa178b4d 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,13 +41,16 @@
>  #include "libavutil/time.h"
>  #include "libavutil/imgutils.h"
>  #include "libavutil/film_grain_params.h"
> +#include "libavutil/stereo3d.h"
>  
>  #include "avcodec.h"
>  #include "codec_internal.h"
>  #include "internal.h"
>  #include "decode.h"
>  #include "hwconfig.h"
> +#include "get_bits.h"
>  #include "qsv.h"
> +#include "h264_sei.h"
>  #include "qsv_internal.h"
>  
>  static const AVRational mfx_tb = { 1, 90000 };
> @@ -101,6 +104,10 @@ typedef struct QSVContext {
>  
>      mfxExtBuffer **ext_buffers;
>      int         nb_ext_buffers;
> +
> +    mfxPayload payload;
> +    H264SEIContext sei;
> +    H264ParamSets ps;
>  } QSVContext;
>  
>  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx,
> mfxExtAV1FilmGrainParam
>  }
>  #endif
>  
> +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
> +{
> +    if (!fpa || !frame)
> +        return AVERROR(EINVAL);
> +
> +    if (!fpa->arrangement_cancel_flag &&
> +        fpa->arrangement_type <= 6 &&
> +        fpa->content_interpretation_type > 0 &&
> +        fpa->content_interpretation_type < 3) {
> +        AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
> +        if (stereo) {
> +            switch (fpa->arrangement_type) {
> +            case 0:
> +                stereo->type = AV_STEREO3D_CHECKERBOARD;
> +                break;
> +            case 1:
> +                stereo->type = AV_STEREO3D_COLUMNS;
> +                break;
> +            case 2:
> +                stereo->type = AV_STEREO3D_LINES;
> +                break;
> +            case 3:
> +                if (fpa->quincunx_sampling_flag)
> +                    stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
> +                else
> +                    stereo->type = AV_STEREO3D_SIDEBYSIDE;
> +                break;
> +            case 4:
> +                stereo->type = AV_STEREO3D_TOPBOTTOM;
> +                break;
> +            case 5:
> +                stereo->type = AV_STEREO3D_FRAMESEQUENCE;
> +                if (fpa->current_frame_is_frame0_flag)
> +                    stereo->view = AV_STEREO3D_VIEW_LEFT;
> +                else
> +                    stereo->view = AV_STEREO3D_VIEW_RIGHT;
> +                break;
> +            case 6:
> +                stereo->type = AV_STEREO3D_2D;
> +                break;
> +            }
> +
> +            if (fpa->content_interpretation_type == 2)
> +                stereo->flags = AV_STEREO3D_FLAG_INVERT;
> +        }
> +    }
> +    return 0;
> +}
> +
> +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame
> *frame)
> +{
> +    GetBitContext gb_payload;
> +    uint8_t *sei_buffer;
> +    int sei_buffer_index;
> +    int ret;
> +
> +    if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
> +        return 0;
> +
> +    sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
> +    if (!sei_buffer) {
> +        av_freep(&sei_buffer);
> +        return AVERROR(ENOMEM);
> +    }
> +    /* remove emulation prevention bytes */
> +    sei_buffer_index = 0;
> +    for (int i = 0; i < q->payload.NumBit / 8; i++) {
> +        if (q->payload.Data[i] == 3)
> +            i++;
> +        sei_buffer[sei_buffer_index] = q->payload.Data[i];
> +        sei_buffer_index += 1;
> +    }
> +
> +    ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);
> +    if (ret < 0) {
> +        av_freep(&sei_buffer);
> +        return ret;
> +    }
> +
> +    ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
> +    if (ret < 0) {
> +        av_freep(&sei_buffer);
> +        return ret;
> +    }
> +
> +    switch (q->payload.Type) {
> +    case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
> +        ret = h264_decode_fpa(&q->sei.frame_packing, frame);
> +        break;
> +    default:
> +        break;
> +    }
> +
> +    av_freep(&sei_buffer);
> +    return ret;
> +}
> +
> +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q,
> AVFrame *frame)
> +{
> +    mfxU64 ts;
> +    mfxStatus sts;
> +    int ret = 0;
> +
> +    if (q->payload.BufSize == 0) {
> +        q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
> +        if (!q->payload.Data)
> +            return AVERROR(ENOMEM);
> +        q->payload.BufSize = QSV_PAYLOAD_SIZE;
> +    }
> +
> +    sts = MFX_ERR_NONE;
> +    while (sts == MFX_ERR_NONE) {
> +
> +        sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
> +        if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
> +                                          "Realloc buffer\n");
> +            if (q->payload.BufSize >= INT16_MAX / 2)
> +                return AVERROR(ENOMEM);
> +            q->payload.BufSize = q->payload.BufSize * 2;
> +            av_freep(&q->payload.Data);
> +            q->payload.Data = av_mallocz(q->payload.BufSize);
> +            if (!q->payload.Data)
> +                return AVERROR(ENOMEM);
> +            continue;
> +        } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
> +            break;
> +        }
> +
> +        switch (avctx->codec_id) {
> +        case AV_CODEC_ID_H264:
> +            ret = h264_parse_side_data(avctx, q, frame);
> +            break;
> +        default:
> +            break;
> +        }
> +        if (ret < 0) {
> +            av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
> +            break;
> +        }
> +    }
> +    return ret;
> +}
> +
>  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>                        AVFrame *frame, int *got_frame,
>                        const AVPacket *avpkt)
> @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>  
>          outsurf = &aframe.frame->surface;
>  
> +        ret = extract_frame_side_data(avctx, q, frame);
> +        if (ret == AVERROR_INVALIDDATA)
> +            av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
> +        else if (ret < 0) {
> +            av_log(avctx, AV_LOG_ERROR, "Error extracting side data from
> packet\n");
> +            return ret;
> +        }
> +

The payload got from the SDK is in encoding order, however the frame returned
from MFXVideoDECODE_DecodeFrameAsync() is in display order, see the discussion
about MFXVideoDECODE_GetPayload() in 
https://github.com/Intel-Media-SDK/MediaSDK/issues/2597 . I think it would be
better to ask the SDK returns the corresponding info, like what they did in 
https://github.com/Intel-Media-SDK/MediaSDK/pull/2726 .

Thanks
Haihao



>          frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx-
> >pkt_timebase);
>  #if QSV_VERSION_ATLEAST(1, 34)
>          if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
> @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q)
>      av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
>      av_buffer_unref(&q->frames_ctx.mids_buf);
>      av_buffer_pool_uninit(&q->pool);
> +    av_freep(&q->payload.Data);
>  }
>  
>  static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
Andreas Rheinhardt April 28, 2022, 1:14 a.m. UTC | #2 
Wenbin Chen:
> Use h264_sei to parse SEI data got from MediaSDK. Extract frame
> packing arrangement information from SEI data and config AVStereo3D
> side data for decoded frame.
> 
> Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> Signed-off-by: Tong Wu <tong1.wu@intel.com>
> ---
>  libavcodec/qsv_internal.h |   2 +
>  libavcodec/qsvdec.c       | 160 ++++++++++++++++++++++++++++++++++++++
>  2 files changed, 162 insertions(+)
> 
> diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> index e2aecdcbd6..a804c392c1 100644
> --- a/libavcodec/qsv_internal.h
> +++ b/libavcodec/qsv_internal.h
> @@ -54,6 +54,8 @@
>  
>  #define QSV_MAX_FRAME_EXT_PARAMS 4
>  
> +#define QSV_PAYLOAD_SIZE 1024
> +
>  #define QSV_VERSION_ATLEAST(MAJOR, MINOR)   \
>      (MFX_VERSION_MAJOR > (MAJOR) ||         \
>       MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 5fc5bed4c8..26fa178b4d 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,13 +41,16 @@
>  #include "libavutil/time.h"
>  #include "libavutil/imgutils.h"
>  #include "libavutil/film_grain_params.h"
> +#include "libavutil/stereo3d.h"
>  
>  #include "avcodec.h"
>  #include "codec_internal.h"
>  #include "internal.h"
>  #include "decode.h"
>  #include "hwconfig.h"
> +#include "get_bits.h"
>  #include "qsv.h"
> +#include "h264_sei.h"
>  #include "qsv_internal.h"
>  
>  static const AVRational mfx_tb = { 1, 90000 };
> @@ -101,6 +104,10 @@ typedef struct QSVContext {
>  
>      mfxExtBuffer **ext_buffers;
>      int         nb_ext_buffers;
> +
> +    mfxPayload payload;
> +    H264SEIContext sei;
> +    H264ParamSets ps;
>  } QSVContext;
>  
>  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
>  }
>  #endif
>  
> +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
> +{
> +    if (!fpa || !frame)
> +        return AVERROR(EINVAL);
> +
> +    if (!fpa->arrangement_cancel_flag &&
> +        fpa->arrangement_type <= 6 &&
> +        fpa->content_interpretation_type > 0 &&
> +        fpa->content_interpretation_type < 3) {
> +        AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
> +        if (stereo) {
> +            switch (fpa->arrangement_type) {
> +            case 0:
> +                stereo->type = AV_STEREO3D_CHECKERBOARD;
> +                break;
> +            case 1:
> +                stereo->type = AV_STEREO3D_COLUMNS;
> +                break;
> +            case 2:
> +                stereo->type = AV_STEREO3D_LINES;
> +                break;
> +            case 3:
> +                if (fpa->quincunx_sampling_flag)
> +                    stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
> +                else
> +                    stereo->type = AV_STEREO3D_SIDEBYSIDE;
> +                break;
> +            case 4:
> +                stereo->type = AV_STEREO3D_TOPBOTTOM;
> +                break;
> +            case 5:
> +                stereo->type = AV_STEREO3D_FRAMESEQUENCE;
> +                if (fpa->current_frame_is_frame0_flag)
> +                    stereo->view = AV_STEREO3D_VIEW_LEFT;
> +                else
> +                    stereo->view = AV_STEREO3D_VIEW_RIGHT;
> +                break;
> +            case 6:
> +                stereo->type = AV_STEREO3D_2D;
> +                break;
> +            }
> +
> +            if (fpa->content_interpretation_type == 2)
> +                stereo->flags = AV_STEREO3D_FLAG_INVERT;
> +        }
> +    }
> +    return 0;
> +}
> +
> +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
> +{
> +    GetBitContext gb_payload;
> +    uint8_t *sei_buffer;
> +    int sei_buffer_index;
> +    int ret;
> +
> +    if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
> +        return 0;
> +
> +    sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
> +    if (!sei_buffer) {
> +        av_freep(&sei_buffer);
> +        return AVERROR(ENOMEM);
> +    }
> +    /* remove emulation prevention bytes */
> +    sei_buffer_index = 0;
> +    for (int i = 0; i < q->payload.NumBit / 8; i++) {
> +        if (q->payload.Data[i] == 3)
> +            i++;
> +        sei_buffer[sei_buffer_index] = q->payload.Data[i];
> +        sei_buffer_index += 1;

Not every 0x03 is an emulation prevention byte.

> +    }
> +
> +    ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);

Buffers used with the get-bits API need to be padded. Best to use
av_fast_padded_malloc and ff_h2645_extract_rbsp.

> +    if (ret < 0) {
> +        av_freep(&sei_buffer);
> +        return ret;
> +    }
> +
> +    ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
> +    if (ret < 0) {
> +        av_freep(&sei_buffer);
> +        return ret;
> +    }
> +
> +    switch (q->payload.Type) {
> +    case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
> +        ret = h264_decode_fpa(&q->sei.frame_packing, frame);
> +        break;
> +    default:
> +        break;
> +    }
> +
> +    av_freep(&sei_buffer);
> +    return ret;
> +}
> +
> +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
> +{
> +    mfxU64 ts;
> +    mfxStatus sts;
> +    int ret = 0;
> +
> +    if (q->payload.BufSize == 0) {
> +        q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
> +        if (!q->payload.Data)
> +            return AVERROR(ENOMEM);
> +        q->payload.BufSize = QSV_PAYLOAD_SIZE;
> +    }
> +
> +    sts = MFX_ERR_NONE;
> +    while (sts == MFX_ERR_NONE) {
> +
> +        sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
> +        if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
> +            av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
> +                                          "Realloc buffer\n");

Why this warning? Why a log-message at all?

> +            if (q->payload.BufSize >= INT16_MAX / 2)

Where does this limit come from?

> +                return AVERROR(ENOMEM);
> +            q->payload.BufSize = q->payload.BufSize * 2;
> +            av_freep(&q->payload.Data);
> +            q->payload.Data = av_mallocz(q->payload.BufSize);
> +            if (!q->payload.Data)
> +                return AVERROR(ENOMEM);

q->payload.Data is NULL if this happens, but q->payload.BufSize will be
> 0, so that q->payload used in the next MFXVideoDECODE_GetPayload call
will be inconsistent.

> +            continue;
> +        } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
> +            break;
> +        }
> +
> +        switch (avctx->codec_id) {
> +        case AV_CODEC_ID_H264:
> +            ret = h264_parse_side_data(avctx, q, frame);
> +            break;
> +        default:
> +            break;
> +        }
> +        if (ret < 0) {
> +            av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
> +            break;
> +        }
> +    }
> +    return ret;
> +}
> +
>  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>                        AVFrame *frame, int *got_frame,
>                        const AVPacket *avpkt)
> @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>  
>          outsurf = &aframe.frame->surface;
>  
> +        ret = extract_frame_side_data(avctx, q, frame);
> +        if (ret == AVERROR_INVALIDDATA)
> +            av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
> +        else if (ret < 0) {
> +            av_log(avctx, AV_LOG_ERROR, "Error extracting side data from packet\n");
> +            return ret;
> +        }
> +
>          frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase);
>  #if QSV_VERSION_ATLEAST(1, 34)
>          if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
> @@ -770,6 +929,7 @@ static void qsv_decode_close_qsvcontext(QSVContext *q)
>      av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
>      av_buffer_unref(&q->frames_ctx.mids_buf);
>      av_buffer_pool_uninit(&q->pool);
> +    av_freep(&q->payload.Data);
>  }
>  
>  static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
Chen, Wenbin April 28, 2022, 1:37 a.m. UTC | #3 
> On Tue, 2022-04-26 at 18:00 +0800, Wenbin Chen wrote:
> > Use h264_sei to parse SEI data got from MediaSDK. Extract frame
> > packing arrangement information from SEI data and config AVStereo3D
> > side data for decoded frame.
> >
> > Signed-off-by: Wenbin Chen <wenbin.chen@intel.com>
> > Signed-off-by: Tong Wu <tong1.wu@intel.com>
> > ---
> >  libavcodec/qsv_internal.h |   2 +
> >  libavcodec/qsvdec.c       | 160
> ++++++++++++++++++++++++++++++++++++++
> >  2 files changed, 162 insertions(+)
> >
> > diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> > index e2aecdcbd6..a804c392c1 100644
> > --- a/libavcodec/qsv_internal.h
> > +++ b/libavcodec/qsv_internal.h
> > @@ -54,6 +54,8 @@
> >
> >  #define QSV_MAX_FRAME_EXT_PARAMS 4
> >
> > +#define QSV_PAYLOAD_SIZE 1024
> > +
> >  #define QSV_VERSION_ATLEAST(MAJOR, MINOR)   \
> >      (MFX_VERSION_MAJOR > (MAJOR) ||         \
> >       MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >=
> (MINOR))
> > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> > index 5fc5bed4c8..26fa178b4d 100644
> > --- a/libavcodec/qsvdec.c
> > +++ b/libavcodec/qsvdec.c
> > @@ -41,13 +41,16 @@
> >  #include "libavutil/time.h"
> >  #include "libavutil/imgutils.h"
> >  #include "libavutil/film_grain_params.h"
> > +#include "libavutil/stereo3d.h"
> >
> >  #include "avcodec.h"
> >  #include "codec_internal.h"
> >  #include "internal.h"
> >  #include "decode.h"
> >  #include "hwconfig.h"
> > +#include "get_bits.h"
> >  #include "qsv.h"
> > +#include "h264_sei.h"
> >  #include "qsv_internal.h"
> >
> >  static const AVRational mfx_tb = { 1, 90000 };
> > @@ -101,6 +104,10 @@ typedef struct QSVContext {
> >
> >      mfxExtBuffer **ext_buffers;
> >      int         nb_ext_buffers;
> > +
> > +    mfxPayload payload;
> > +    H264SEIContext sei;
> > +    H264ParamSets ps;
> >  } QSVContext;
> >
> >  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> > @@ -600,6 +607,150 @@ static int qsv_export_film_grain(AVCodecContext
> *avctx,
> > mfxExtAV1FilmGrainParam
> >  }
> >  #endif
> >
> > +static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
> > +{
> > +    if (!fpa || !frame)
> > +        return AVERROR(EINVAL);
> > +
> > +    if (!fpa->arrangement_cancel_flag &&
> > +        fpa->arrangement_type <= 6 &&
> > +        fpa->content_interpretation_type > 0 &&
> > +        fpa->content_interpretation_type < 3) {
> > +        AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
> > +        if (stereo) {
> > +            switch (fpa->arrangement_type) {
> > +            case 0:
> > +                stereo->type = AV_STEREO3D_CHECKERBOARD;
> > +                break;
> > +            case 1:
> > +                stereo->type = AV_STEREO3D_COLUMNS;
> > +                break;
> > +            case 2:
> > +                stereo->type = AV_STEREO3D_LINES;
> > +                break;
> > +            case 3:
> > +                if (fpa->quincunx_sampling_flag)
> > +                    stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
> > +                else
> > +                    stereo->type = AV_STEREO3D_SIDEBYSIDE;
> > +                break;
> > +            case 4:
> > +                stereo->type = AV_STEREO3D_TOPBOTTOM;
> > +                break;
> > +            case 5:
> > +                stereo->type = AV_STEREO3D_FRAMESEQUENCE;
> > +                if (fpa->current_frame_is_frame0_flag)
> > +                    stereo->view = AV_STEREO3D_VIEW_LEFT;
> > +                else
> > +                    stereo->view = AV_STEREO3D_VIEW_RIGHT;
> > +                break;
> > +            case 6:
> > +                stereo->type = AV_STEREO3D_2D;
> > +                break;
> > +            }
> > +
> > +            if (fpa->content_interpretation_type == 2)
> > +                stereo->flags = AV_STEREO3D_FLAG_INVERT;
> > +        }
> > +    }
> > +    return 0;
> > +}
> > +
> > +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q,
> AVFrame
> > *frame)
> > +{
> > +    GetBitContext gb_payload;
> > +    uint8_t *sei_buffer;
> > +    int sei_buffer_index;
> > +    int ret;
> > +
> > +    if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
> > +        return 0;
> > +
> > +    sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
> > +    if (!sei_buffer) {
> > +        av_freep(&sei_buffer);
> > +        return AVERROR(ENOMEM);
> > +    }
> > +    /* remove emulation prevention bytes */
> > +    sei_buffer_index = 0;
> > +    for (int i = 0; i < q->payload.NumBit / 8; i++) {
> > +        if (q->payload.Data[i] == 3)
> > +            i++;
> > +        sei_buffer[sei_buffer_index] = q->payload.Data[i];
> > +        sei_buffer_index += 1;
> > +    }
> > +
> > +    ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);
> > +    if (ret < 0) {
> > +        av_freep(&sei_buffer);
> > +        return ret;
> > +    }
> > +
> > +    ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
> > +    if (ret < 0) {
> > +        av_freep(&sei_buffer);
> > +        return ret;
> > +    }
> > +
> > +    switch (q->payload.Type) {
> > +    case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
> > +        ret = h264_decode_fpa(&q->sei.frame_packing, frame);
> > +        break;
> > +    default:
> > +        break;
> > +    }
> > +
> > +    av_freep(&sei_buffer);
> > +    return ret;
> > +}
> > +
> > +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext
> *q,
> > AVFrame *frame)
> > +{
> > +    mfxU64 ts;
> > +    mfxStatus sts;
> > +    int ret = 0;
> > +
> > +    if (q->payload.BufSize == 0) {
> > +        q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
> > +        if (!q->payload.Data)
> > +            return AVERROR(ENOMEM);
> > +        q->payload.BufSize = QSV_PAYLOAD_SIZE;
> > +    }
> > +
> > +    sts = MFX_ERR_NONE;
> > +    while (sts == MFX_ERR_NONE) {
> > +
> > +        sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
> > +        if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
> > +            av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
> > +                                          "Realloc buffer\n");
> > +            if (q->payload.BufSize >= INT16_MAX / 2)
> > +                return AVERROR(ENOMEM);
> > +            q->payload.BufSize = q->payload.BufSize * 2;
> > +            av_freep(&q->payload.Data);
> > +            q->payload.Data = av_mallocz(q->payload.BufSize);
> > +            if (!q->payload.Data)
> > +                return AVERROR(ENOMEM);
> > +            continue;
> > +        } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
> > +            break;
> > +        }
> > +
> > +        switch (avctx->codec_id) {
> > +        case AV_CODEC_ID_H264:
> > +            ret = h264_parse_side_data(avctx, q, frame);
> > +            break;
> > +        default:
> > +            break;
> > +        }
> > +        if (ret < 0) {
> > +            av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
> > +            break;
> > +        }
> > +    }
> > +    return ret;
> > +}
> > +
> >  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
> >                        AVFrame *frame, int *got_frame,
> >                        const AVPacket *avpkt)
> > @@ -709,6 +860,14 @@ static int qsv_decode(AVCodecContext *avctx,
> QSVContext
> > *q,
> >
> >          outsurf = &aframe.frame->surface;
> >
> > +        ret = extract_frame_side_data(avctx, q, frame);
> > +        if (ret == AVERROR_INVALIDDATA)
> > +            av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
> > +        else if (ret < 0) {
> > +            av_log(avctx, AV_LOG_ERROR, "Error extracting side data from
> > packet\n");
> > +            return ret;
> > +        }
> > +
> 
> The payload got from the SDK is in encoding order, however the frame
> returned
> from MFXVideoDECODE_DecodeFrameAsync() is in display order, see the
> discussion
> about MFXVideoDECODE_GetPayload() in
> https://github.com/Intel-Media-SDK/MediaSDK/issues/2597 . I think it would
> be
> better to ask the SDK returns the corresponding info, like what they did in
> https://github.com/Intel-Media-SDK/MediaSDK/pull/2726 .
> 
> Thanks
> Haihao

Thanks for your information. I will try to fix this problem.

Thanks
Wenbin
> 
> 
> 
> >          frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx-
> > >pkt_timebase);
> >  #if QSV_VERSION_ATLEAST(1, 34)
> >          if ((avctx->export_side_data &
> AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
> > @@ -770,6 +929,7 @@ static void
> qsv_decode_close_qsvcontext(QSVContext *q)
> >      av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
> >      av_buffer_unref(&q->frames_ctx.mids_buf);
> >      av_buffer_pool_uninit(&q->pool);
> > +    av_freep(&q->payload.Data);
> >  }
> >
> >  static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff mbox series

Patch

diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
index e2aecdcbd6..a804c392c1 100644
--- a/libavcodec/qsv_internal.h
+++ b/libavcodec/qsv_internal.h
@@ -54,6 +54,8 @@ 
 
 #define QSV_MAX_FRAME_EXT_PARAMS 4
 
+#define QSV_PAYLOAD_SIZE 1024
+
 #define QSV_VERSION_ATLEAST(MAJOR, MINOR)   \
     (MFX_VERSION_MAJOR > (MAJOR) ||         \
      MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR))
diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 5fc5bed4c8..26fa178b4d 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -41,13 +41,16 @@ 
 #include "libavutil/time.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/film_grain_params.h"
+#include "libavutil/stereo3d.h"
 
 #include "avcodec.h"
 #include "codec_internal.h"
 #include "internal.h"
 #include "decode.h"
 #include "hwconfig.h"
+#include "get_bits.h"
 #include "qsv.h"
+#include "h264_sei.h"
 #include "qsv_internal.h"
 
 static const AVRational mfx_tb = { 1, 90000 };
@@ -101,6 +104,10 @@  typedef struct QSVContext {
 
     mfxExtBuffer **ext_buffers;
     int         nb_ext_buffers;
+
+    mfxPayload payload;
+    H264SEIContext sei;
+    H264ParamSets ps;
 } QSVContext;
 
 static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
@@ -600,6 +607,150 @@  static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
 }
 #endif
 
+static int h264_decode_fpa(H264SEIFramePacking *fpa, AVFrame *frame)
+{
+    if (!fpa || !frame)
+        return AVERROR(EINVAL);
+
+    if (!fpa->arrangement_cancel_flag &&
+        fpa->arrangement_type <= 6 &&
+        fpa->content_interpretation_type > 0 &&
+        fpa->content_interpretation_type < 3) {
+        AVStereo3D *stereo = av_stereo3d_create_side_data(frame);
+        if (stereo) {
+            switch (fpa->arrangement_type) {
+            case 0:
+                stereo->type = AV_STEREO3D_CHECKERBOARD;
+                break;
+            case 1:
+                stereo->type = AV_STEREO3D_COLUMNS;
+                break;
+            case 2:
+                stereo->type = AV_STEREO3D_LINES;
+                break;
+            case 3:
+                if (fpa->quincunx_sampling_flag)
+                    stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
+                else
+                    stereo->type = AV_STEREO3D_SIDEBYSIDE;
+                break;
+            case 4:
+                stereo->type = AV_STEREO3D_TOPBOTTOM;
+                break;
+            case 5:
+                stereo->type = AV_STEREO3D_FRAMESEQUENCE;
+                if (fpa->current_frame_is_frame0_flag)
+                    stereo->view = AV_STEREO3D_VIEW_LEFT;
+                else
+                    stereo->view = AV_STEREO3D_VIEW_RIGHT;
+                break;
+            case 6:
+                stereo->type = AV_STEREO3D_2D;
+                break;
+            }
+
+            if (fpa->content_interpretation_type == 2)
+                stereo->flags = AV_STEREO3D_FLAG_INVERT;
+        }
+    }
+    return 0;
+}
+
+static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
+{
+    GetBitContext gb_payload;
+    uint8_t *sei_buffer;
+    int sei_buffer_index;
+    int ret;
+
+    if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT)
+        return 0;
+
+    sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8);
+    if (!sei_buffer) {
+        av_freep(&sei_buffer);
+        return AVERROR(ENOMEM);
+    }
+    /* remove emulation prevention bytes */
+    sei_buffer_index = 0;
+    for (int i = 0; i < q->payload.NumBit / 8; i++) {
+        if (q->payload.Data[i] == 3)
+            i++;
+        sei_buffer[sei_buffer_index] = q->payload.Data[i];
+        sei_buffer_index += 1;
+    }
+
+    ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1);
+    if (ret < 0) {
+        av_freep(&sei_buffer);
+        return ret;
+    }
+
+    ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx);
+    if (ret < 0) {
+        av_freep(&sei_buffer);
+        return ret;
+    }
+
+    switch (q->payload.Type) {
+    case SEI_TYPE_FRAME_PACKING_ARRANGEMENT:
+        ret = h264_decode_fpa(&q->sei.frame_packing, frame);
+        break;
+    default:
+        break;
+    }
+
+    av_freep(&sei_buffer);
+    return ret;
+}
+
+static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame)
+{
+    mfxU64 ts;
+    mfxStatus sts;
+    int ret = 0;
+
+    if (q->payload.BufSize == 0) {
+        q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE);
+        if (!q->payload.Data)
+            return AVERROR(ENOMEM);
+        q->payload.BufSize = QSV_PAYLOAD_SIZE;
+    }
+
+    sts = MFX_ERR_NONE;
+    while (sts == MFX_ERR_NONE) {
+
+        sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload);
+        if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) {
+            av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough."
+                                          "Realloc buffer\n");
+            if (q->payload.BufSize >= INT16_MAX / 2)
+                return AVERROR(ENOMEM);
+            q->payload.BufSize = q->payload.BufSize * 2;
+            av_freep(&q->payload.Data);
+            q->payload.Data = av_mallocz(q->payload.BufSize);
+            if (!q->payload.Data)
+                return AVERROR(ENOMEM);
+            continue;
+        } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) {
+            break;
+        }
+
+        switch (avctx->codec_id) {
+        case AV_CODEC_ID_H264:
+            ret = h264_parse_side_data(avctx, q, frame);
+            break;
+        default:
+            break;
+        }
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_WARNING, "parse side data failed\n");
+            break;
+        }
+    }
+    return ret;
+}
+
 static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
                       AVFrame *frame, int *got_frame,
                       const AVPacket *avpkt)
@@ -709,6 +860,14 @@  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
 
         outsurf = &aframe.frame->surface;
 
+        ret = extract_frame_side_data(avctx, q, frame);
+        if (ret == AVERROR_INVALIDDATA)
+            av_log(avctx, AV_LOG_WARNING, "Side data is invalid\n");
+        else if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Error extracting side data from packet\n");
+            return ret;
+        }
+
         frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase);
 #if QSV_VERSION_ATLEAST(1, 34)
         if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) &&
@@ -770,6 +929,7 @@  static void qsv_decode_close_qsvcontext(QSVContext *q)
     av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
     av_buffer_unref(&q->frames_ctx.mids_buf);
     av_buffer_pool_uninit(&q->pool);
+    av_freep(&q->payload.Data);
 }
 
 static int qsv_process_data(AVCodecContext *avctx, QSVContext *q,