diff mbox series

[FFmpeg-devel,4/5] avcodec/qsvdec: Implement SEI parsing for QSV decoders

Message ID DM8P223MB03655F44A7BF01132BB2959DBA669@DM8P223MB0365.NAMP223.PROD.OUTLOOK.COM
State New
Headers show
Series [FFmpeg-devel,1/5] avcodec/mpeg12dec: make mpeg_decode_user_data() accessible | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished
andriy/make_ppc success Make finished
andriy/make_fate_ppc success Make fate finished

Commit Message

Soft Works Nov. 29, 2021, 6:51 p.m. UTC
Signed-off-by: softworkz <softworkz@hotmail.com>
---
 libavcodec/qsvdec.c | 210 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 210 insertions(+)

Comments

Xiang, Haihao Dec. 29, 2021, 6:26 a.m. UTC | #1
On Mon, 2021-11-29 at 18:51 +0000, Soft Works wrote:
> Signed-off-by: softworkz <softworkz@hotmail.com>
> ---
>  libavcodec/qsvdec.c | 210 ++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 210 insertions(+)
> 
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 8bce9f2cf0..62f3e65455 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -45,6 +45,12 @@
>  #include "hwconfig.h"
>  #include "qsv.h"
>  #include "qsv_internal.h"
> +#include "h264dec.h"
> +#include "h264_sei.h"
> +#include "hevcdec.h"
> +#include "hevc_ps.h"
> +#include "hevc_sei.h"
> +#include "mpeg12.h"
>  
>  static const AVRational mfx_tb = { 1, 90000 };
>  
> @@ -92,6 +98,8 @@ typedef struct QSVContext {
>  
>      mfxExtBuffer **ext_buffers;
>      int         nb_ext_buffers;
> +
> +    Mpeg1Context mpeg_ctx;
>  } QSVContext;
>  
>  static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
> @@ -498,6 +506,185 @@ static QSVFrame *find_frame(QSVContext *q,
> mfxFrameSurface1 *surf)
>      return NULL;
>  }
>  
> +static int find_start_offset(mfxU8 data[1024])
> +{
> +    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
> +        return 3;
> +
> +    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
> +        return 4;
> +
> +    return 0;
> +}
> +
> +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> +    H264SEIContext sei = { 0 };
> +    GetBitContext gb = { 0 };
> +    mfxU8 data[1024] = { 0 };
> +    mfxPayload payload = { 0, .Data = &data[0], .BufSize = sizeof(data) };
> +    mfxU64 ts;
> +    int ret;
> +
> +    while (1) {
> +        int start;
> +        memset(&data[0], 0, sizeof(data));
> +
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);

The *out* frame returned from MFXVideoDECODE_DecodeFrameAsync is in display
order. Does MFXVideoDECODE_GetPayload get payload in display order too ? I
didn't see such description in 
https://github.com/Intel-Media-SDK/MediaSDK/blob/master/doc/mediasdk-man.md 

Thanks
Haihao


> +        if (ret != MFX_ERR_NONE) {
> +            av_log(avctx, AV_LOG_WARNING, "error getting SEI payload: %d \n",
> ret);
> +            return ret;
> +        }
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= sizeof(data) * 8) {
> +            break;
> +        }
> +
> +        start = find_start_offset(data);
> +
> +        switch (payload.Type) {
> +            case SEI_TYPE_BUFFERING_PERIOD:
> +            case SEI_TYPE_PIC_TIMING:
> +                continue;
> +        }
> +
> +        if (init_get_bits(&gb, &data[start], payload.NumBit - start * 8) < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream
> reader");
> +        else
> +            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
> +
> +        if (ret < 0)
> +            av_log(avctx, AV_LOG_WARNING, "Error parsing SEI type:
> %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +        else
> +            av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n",
> payload.Type, payload.NumBit);
> +    }
> +
> +    if (out)
> +        return ff_h264_export_frame_props(avctx, &sei, NULL, out);
> +
> +    return 0;
> +}
> +
> +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
> +{
> +    HEVCSEI sei = { 0 };
> +    HEVCParamSets ps = { 0 };
> +    GetBitContext gb = { 0 };
> +    mfxU8 data[1024] = { 0 };
> +    mfxPayload payload = { 0, .Data = &data[0], .BufSize = sizeof(data) };
> +    mfxU64 ts;
> +    int ret;
> +
> +    while (1) {
> +        int start;
> +        memset(&data[0], 0, sizeof(data));
> +
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret != MFX_ERR_NONE) {
> +            av_log(avctx, AV_LOG_WARNING, "error getting SEI payload: %d \n",
> ret);
> +            return ret;
> +        }
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= sizeof(data) * 8) {
> +            break;
> +        }
> +
> +        start = find_start_offset(data);
> +
> +        switch (payload.Type) {
> +            case SEI_TYPE_BUFFERING_PERIOD:
> +            case SEI_TYPE_PIC_TIMING:
> +                continue;
> +            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> +            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> +                // There seems to be a bug in MSDK for these
> +                payload.NumBit -= 8;
> +                break;
> +        }
> +
> +        if (init_get_bits(&gb, &data[start], payload.NumBit - start * 8) < 0)
> +            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream
> reader");
> +        else
> +            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps,
> HEVC_NAL_SEI_PREFIX);
> +
> +        if (ret < 0)
> +            av_log(avctx, AV_LOG_WARNING, "error parsing SEI type:
> %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +        else
> +            av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n",
> payload.Type, payload.NumBit);
> +    }
> +
> +    if (out)
> +        return ff_set_side_data(avctx, &sei, NULL, out);
> +
> +    return 0;
> +}
> +
> +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame*
> out)
> +{
> +    Mpeg1Context *mpeg_ctx = &q->mpeg_ctx;
> +    mfxU8 data[1024] = { 0 };
> +    mfxPayload payload = { 0, .Data = &data[0], .BufSize = sizeof(data) };
> +    mfxU64 ts;
> +    int ret;
> +
> +    while (1) {
> +        int start;
> +
> +        memset(&data[0], 0, sizeof(data));
> +        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
> +        if (ret != MFX_ERR_NONE) {
> +            av_log(avctx, AV_LOG_WARNING, "error getting SEI payload: %d \n",
> ret);
> +            return ret;
> +        }
> +
> +        if (payload.NumBit == 0 || payload.NumBit >= sizeof(data) * 8) {
> +            break;
> +        }
> +
> +        start = find_start_offset(data);
> +
> +        start++;
> +
> +        ff_mpeg_decode_user_data(avctx, mpeg_ctx, &data[start],
> (int)((payload.NumBit + 7) / 8) - start);
> +
> +        if (ret < 0)
> +            av_log(avctx, AV_LOG_WARNING, "error parsing SEI type:
> %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
> +        else
> +            av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d
> start %d -> %.s\n", payload.Type, payload.NumBit, start, (char
> *)(&data[start]));
> +    }
> +
> +    if (!out)
> +        return 0;
> +
> +    if (mpeg_ctx->a53_buf_ref) {
> +
> +        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out,
> AV_FRAME_DATA_A53_CC, mpeg_ctx->a53_buf_ref);
> +        if (!sd)
> +            av_buffer_unref(&mpeg_ctx->a53_buf_ref);
> +        mpeg_ctx->a53_buf_ref = NULL;
> +    }
> +
> +    if (mpeg_ctx->has_stereo3d) {
> +        AVStereo3D *stereo = av_stereo3d_create_side_data(out);
> +        if (!stereo)
> +            return AVERROR(ENOMEM);
> +
> +        *stereo = mpeg_ctx->stereo3d;
> +        mpeg_ctx->has_stereo3d = 0;
> +    }
> +
> +    if (mpeg_ctx->has_afd) {
> +        AVFrameSideData *sd = av_frame_new_side_data(out, AV_FRAME_DATA_AFD,
> 1);
> +        if (!sd)
> +            return AVERROR(ENOMEM);
> +
> +        *sd->data   = mpeg_ctx->afd;
> +        mpeg_ctx->has_afd = 0;
> +    }
> +
> +    return 0;
> +}
> +
>  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>                        AVFrame *frame, int *got_frame,
>                        const AVPacket *avpkt)
> @@ -535,6 +722,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>                                                insurf, &outsurf, sync);
>          if (ret == MFX_WRN_DEVICE_BUSY)
>              av_usleep(500);
> +        else if (avctx->codec_id == AV_CODEC_ID_MPEG1VIDEO || avctx->codec_id 
> == AV_CODEC_ID_MPEG2VIDEO)
> +            parse_sei_mpeg12(avctx, q, NULL);
>  
>      } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
>  
> @@ -570,6 +759,27 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>              return AVERROR_BUG;
>          }
>  
> +        switch (avctx->codec_id) {
> +        case AV_CODEC_ID_MPEG1VIDEO:
> +        case AV_CODEC_ID_MPEG2VIDEO:
> +            ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
> +            break;
> +        case AV_CODEC_ID_H264:
> +            ret = parse_sei_h264(avctx, q, out_frame->frame);
> +            break;
> +        case AV_CODEC_ID_HEVC:
> +            ret = parse_sei_hevc(avctx, q, out_frame->frame);
> +            break;
> +        default:
> +            ret = 0;
> +        }
> +
> +        if (ret < 0) {
> +            av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data\n");
> +            av_freep(&sync);
> +            return ret;
> +        }
> +#
>          out_frame->queued = 1;
>          av_fifo_generic_write(q->async_fifo, &out_frame, sizeof(out_frame),
> NULL);
>          av_fifo_generic_write(q->async_fifo,
> &sync,      sizeof(sync),      NULL);
diff mbox series

Patch

diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 8bce9f2cf0..62f3e65455 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -45,6 +45,12 @@ 
 #include "hwconfig.h"
 #include "qsv.h"
 #include "qsv_internal.h"
+#include "h264dec.h"
+#include "h264_sei.h"
+#include "hevcdec.h"
+#include "hevc_ps.h"
+#include "hevc_sei.h"
+#include "mpeg12.h"
 
 static const AVRational mfx_tb = { 1, 90000 };
 
@@ -92,6 +98,8 @@  typedef struct QSVContext {
 
     mfxExtBuffer **ext_buffers;
     int         nb_ext_buffers;
+
+    Mpeg1Context mpeg_ctx;
 } QSVContext;
 
 static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
@@ -498,6 +506,185 @@  static QSVFrame *find_frame(QSVContext *q, mfxFrameSurface1 *surf)
     return NULL;
 }
 
+static int find_start_offset(mfxU8 data[1024])
+{
+    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
+        return 3;
+
+    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
+        return 4;
+
+    return 0;
+}
+
+static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+    H264SEIContext sei = { 0 };
+    GetBitContext gb = { 0 };
+    mfxU8 data[1024] = { 0 };
+    mfxPayload payload = { 0, .Data = &data[0], .BufSize = sizeof(data) };
+    mfxU64 ts;
+    int ret;
+
+    while (1) {
+        int start;
+        memset(&data[0], 0, sizeof(data));
+
+        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+        if (ret != MFX_ERR_NONE) {
+            av_log(avctx, AV_LOG_WARNING, "error getting SEI payload: %d \n", ret);
+            return ret;
+        }
+
+        if (payload.NumBit == 0 || payload.NumBit >= sizeof(data) * 8) {
+            break;
+        }
+
+        start = find_start_offset(data);
+
+        switch (payload.Type) {
+            case SEI_TYPE_BUFFERING_PERIOD:
+            case SEI_TYPE_PIC_TIMING:
+                continue;
+        }
+
+        if (init_get_bits(&gb, &data[start], payload.NumBit - start * 8) < 0)
+            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader");
+        else
+            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
+
+        if (ret < 0)
+            av_log(avctx, AV_LOG_WARNING, "Error parsing SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+        else
+            av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n", payload.Type, payload.NumBit);
+    }
+
+    if (out)
+        return ff_h264_export_frame_props(avctx, &sei, NULL, out);
+
+    return 0;
+}
+
+static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+    HEVCSEI sei = { 0 };
+    HEVCParamSets ps = { 0 };
+    GetBitContext gb = { 0 };
+    mfxU8 data[1024] = { 0 };
+    mfxPayload payload = { 0, .Data = &data[0], .BufSize = sizeof(data) };
+    mfxU64 ts;
+    int ret;
+
+    while (1) {
+        int start;
+        memset(&data[0], 0, sizeof(data));
+
+        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+        if (ret != MFX_ERR_NONE) {
+            av_log(avctx, AV_LOG_WARNING, "error getting SEI payload: %d \n", ret);
+            return ret;
+        }
+
+        if (payload.NumBit == 0 || payload.NumBit >= sizeof(data) * 8) {
+            break;
+        }
+
+        start = find_start_offset(data);
+
+        switch (payload.Type) {
+            case SEI_TYPE_BUFFERING_PERIOD:
+            case SEI_TYPE_PIC_TIMING:
+                continue;
+            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
+            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
+                // There seems to be a bug in MSDK for these
+                payload.NumBit -= 8;
+                break;
+        }
+
+        if (init_get_bits(&gb, &data[start], payload.NumBit - start * 8) < 0)
+            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader");
+        else
+            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX);
+
+        if (ret < 0)
+            av_log(avctx, AV_LOG_WARNING, "error parsing SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+        else
+            av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n", payload.Type, payload.NumBit);
+    }
+
+    if (out)
+        return ff_set_side_data(avctx, &sei, NULL, out);
+
+    return 0;
+}
+
+static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+    Mpeg1Context *mpeg_ctx = &q->mpeg_ctx;
+    mfxU8 data[1024] = { 0 };
+    mfxPayload payload = { 0, .Data = &data[0], .BufSize = sizeof(data) };
+    mfxU64 ts;
+    int ret;
+
+    while (1) {
+        int start;
+
+        memset(&data[0], 0, sizeof(data));
+        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+        if (ret != MFX_ERR_NONE) {
+            av_log(avctx, AV_LOG_WARNING, "error getting SEI payload: %d \n", ret);
+            return ret;
+        }
+
+        if (payload.NumBit == 0 || payload.NumBit >= sizeof(data) * 8) {
+            break;
+        }
+
+        start = find_start_offset(data);
+
+        start++;
+
+        ff_mpeg_decode_user_data(avctx, mpeg_ctx, &data[start], (int)((payload.NumBit + 7) / 8) - start);
+
+        if (ret < 0)
+            av_log(avctx, AV_LOG_WARNING, "error parsing SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+        else
+            av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char *)(&data[start]));
+    }
+
+    if (!out)
+        return 0;
+
+    if (mpeg_ctx->a53_buf_ref) {
+
+        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, mpeg_ctx->a53_buf_ref);
+        if (!sd)
+            av_buffer_unref(&mpeg_ctx->a53_buf_ref);
+        mpeg_ctx->a53_buf_ref = NULL;
+    }
+
+    if (mpeg_ctx->has_stereo3d) {
+        AVStereo3D *stereo = av_stereo3d_create_side_data(out);
+        if (!stereo)
+            return AVERROR(ENOMEM);
+
+        *stereo = mpeg_ctx->stereo3d;
+        mpeg_ctx->has_stereo3d = 0;
+    }
+
+    if (mpeg_ctx->has_afd) {
+        AVFrameSideData *sd = av_frame_new_side_data(out, AV_FRAME_DATA_AFD, 1);
+        if (!sd)
+            return AVERROR(ENOMEM);
+
+        *sd->data   = mpeg_ctx->afd;
+        mpeg_ctx->has_afd = 0;
+    }
+
+    return 0;
+}
+
 static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
                       AVFrame *frame, int *got_frame,
                       const AVPacket *avpkt)
@@ -535,6 +722,8 @@  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
                                               insurf, &outsurf, sync);
         if (ret == MFX_WRN_DEVICE_BUSY)
             av_usleep(500);
+        else if (avctx->codec_id == AV_CODEC_ID_MPEG1VIDEO || avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
+            parse_sei_mpeg12(avctx, q, NULL);
 
     } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
 
@@ -570,6 +759,27 @@  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
             return AVERROR_BUG;
         }
 
+        switch (avctx->codec_id) {
+        case AV_CODEC_ID_MPEG1VIDEO:
+        case AV_CODEC_ID_MPEG2VIDEO:
+            ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
+            break;
+        case AV_CODEC_ID_H264:
+            ret = parse_sei_h264(avctx, q, out_frame->frame);
+            break;
+        case AV_CODEC_ID_HEVC:
+            ret = parse_sei_hevc(avctx, q, out_frame->frame);
+            break;
+        default:
+            ret = 0;
+        }
+
+        if (ret < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data\n");
+            av_freep(&sync);
+            return ret;
+        }
+#
         out_frame->queued = 1;
         av_fifo_generic_write(q->async_fifo, &out_frame, sizeof(out_frame), NULL);
         av_fifo_generic_write(q->async_fifo, &sync,      sizeof(sync),      NULL);