| Message ID | 61626ebb78fcf603f58c31983fa5916c6ebeaafe.1666670616.git.ffmpegagent@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | Implement SEI parsing for QSV decoders | expand |
| Context | Check | Description |
|---|---|---|
| yinshiyou/make_loongarch64 | success | Make finished |
| yinshiyou/make_fate_loongarch64 | success | Make fate finished |
On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote: > From: softworkz <softworkz@hotmail.com> > > Signed-off-by: softworkz <softworkz@hotmail.com> > --- > libavcodec/Makefile | 2 +- > libavcodec/qsvdec.c | 321 ++++++++++++++++++++++++++++++++++++++++++++ > 2 files changed, 322 insertions(+), 1 deletion(-) > > diff --git a/libavcodec/Makefile b/libavcodec/Makefile > index 90c7f113a3..cbddbb0ace 100644 > --- a/libavcodec/Makefile > +++ b/libavcodec/Makefile > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o > OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o > OBJS-$(CONFIG_QPELDSP) += qpeldsp.o > OBJS-$(CONFIG_QSV) += qsv.o > -OBJS-$(CONFIG_QSVDEC) += qsvdec.o > +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o hevc_sei.o > OBJS-$(CONFIG_QSVENC) += qsvenc.o > OBJS-$(CONFIG_RANGECODER) += rangecoder.o > OBJS-$(CONFIG_RDFT) += rdft.o > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c > index 73405b5747..467a248224 100644 > --- a/libavcodec/qsvdec.c > +++ b/libavcodec/qsvdec.c > @@ -41,6 +41,7 @@ > #include "libavutil/time.h" > #include "libavutil/imgutils.h" > #include "libavutil/film_grain_params.h" > +#include <libavutil/reverse.h> > > #include "avcodec.h" > #include "codec_internal.h" > @@ -49,6 +50,9 @@ > #include "hwconfig.h" > #include "qsv.h" > #include "qsv_internal.h" > +#include "h264_sei.h" > +#include "hevc_ps.h" > +#include "hevc_sei.h" > > #if QSV_ONEVPL > #include <mfxdispatcher.h> > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 }; > AV_NOPTS_VALUE : pts_tb.num ? \ > av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts) > > +#define PAYLOAD_BUFFER_SIZE 65535 > + > typedef struct QSVAsyncFrame { > mfxSyncPoint *sync; > QSVFrame *frame; > @@ -107,6 +113,9 @@ typedef struct QSVContext { > > mfxExtBuffer **ext_buffers; > int nb_ext_buffers; > + > + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE]; > + AVBufferRef *a53_buf_ref; > } QSVContext; > > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { > @@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx, > mfxExtAV1FilmGrainParam > } > #endif > > +static int find_start_offset(mfxU8 data[4]) > +{ > + if (data[0] == 0 && data[1] == 0 && data[2] == 1) > + return 3; > + > + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) > + return 4; > + > + return 0; > +} > + > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out) > +{ > + H264SEIContext sei = { 0 }; > + GetBitContext gb = { 0 }; > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; > + mfxU64 ts; > + int ret; > + > + while (1) { > + int start; > + memset(payload.Data, 0, payload.BufSize); > + > + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), > payload.BufSize); > + return 0; > + } > + if (ret != MFX_ERR_NONE) > + return ret; > + > + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8) > + break; > + > + start = find_start_offset(payload.Data); > + > + switch (payload.Type) { > + case SEI_TYPE_BUFFERING_PERIOD: > + case SEI_TYPE_PIC_TIMING: > + continue; > + } > + > + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * > 8) < 0) > + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader > SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > + else { > + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx); > + > + if (ret < 0) > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > + else > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits > %d\n", payload.Type, payload.NumBit); > + } > + } > + > + if (out) > + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0); > + > + return 0; > +} > + > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* > out) > +{ > + HEVCSEI sei = { 0 }; > + HEVCParamSets ps = { 0 }; > + GetBitContext gb = { 0 }; > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; > + mfxFrameSurface1 *surface = &out->surface; > + mfxU64 ts; > + int ret, has_logged = 0; > + > + while (1) { > + int start; > + memset(payload.Data, 0, payload.BufSize); > + > + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), > payload.BufSize); > + return 0; > + } > + if (ret != MFX_ERR_NONE) > + return ret; > + > + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8) > + break; > + > + if (!has_logged) { > + has_logged = 1; > + av_log(avctx, AV_LOG_VERBOSE, "-------------------------------- > ---------\n"); > + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload > timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp); > + } > + > + if (ts != surface->Data.TimeStamp) { > + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does > not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp); > + } > + > + start = find_start_offset(payload.Data); > + > + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits > %3d Start: %d\n", payload.Type, payload.NumBit, start); > + > + switch (payload.Type) { > + case SEI_TYPE_BUFFERING_PERIOD: > + case SEI_TYPE_PIC_TIMING: > + continue; > + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME: > + // There seems to be a bug in MSDK > + payload.NumBit -= 8; > + > + break; > + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO: > + // There seems to be a bug in MSDK > + payload.NumBit = 48; > + > + break; > + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35: > + // There seems to be a bug in MSDK > + if (payload.NumBit == 552) > + payload.NumBit = 528; > + break; > + } > + > + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * > 8) < 0) > + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader > SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > + else { > + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, > HEVC_NAL_SEI_PREFIX); > + > + if (ret < 0) > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > + else > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits > %d\n", payload.Type, payload.NumBit); > + } > + } > + > + if (has_logged) { > + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n"); > + } > + > + if (out && out->frame) > + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx- > >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma); I got segfault when trying your patchset, Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault. 0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 777 return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma); (gdb) bt #0 0x00007ffff67c0497 in parse_sei_hevc (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 #1 0x00007ffff67c1afe in qsv_decode (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, frame=frame@entry=0x5555556df740, got_frame=got_frame@entry=0x7fffffffd6bc, avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020 BTW the SDK provides support for hevc HDR metadata, we needn't parse SEI payload in qsvdec and may get the corresponding info from the SDK, see https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html Thanks Haihao > + > + return 0; > +} > + > +#define A53_MAX_CC_COUNT 2000 > + > +static int mpeg_decode_a53_cc(AVCodecContext *avctx, QSVContext *s, > + const uint8_t *p, int buf_size) > +{ > + if (buf_size >= 6 && > + p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' && > + p[4] == 3 && (p[5] & 0x40)) { > + /* extract A53 Part 4 CC data */ > + unsigned cc_count = p[5] & 0x1f; > + if (cc_count > 0 && buf_size >= 7 + cc_count * 3) { > + const uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : > 0; > + const uint64_t new_size = (old_size + cc_count > + * UINT64_C(3)); > + int ret; > + > + if (new_size > 3*A53_MAX_CC_COUNT) > + return AVERROR(EINVAL); > + > + ret = av_buffer_realloc(&s->a53_buf_ref, new_size); > + if (ret >= 0) > + memcpy(s->a53_buf_ref->data + old_size, p + 7, cc_count * > UINT64_C(3)); > + > + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS; > + } > + return 1; > + } else if (buf_size >= 2 && p[0] == 0x03 && (p[1]&0x7f) == 0x01) { > + /* extract SCTE-20 CC data */ > + GetBitContext gb; > + unsigned cc_count = 0; > + int ret; > + > + init_get_bits8(&gb, p + 2, buf_size - 2); > + cc_count = get_bits(&gb, 5); > + if (cc_count > 0) { > + uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0; > + uint64_t new_size = (old_size + cc_count * UINT64_C(3)); > + if (new_size > 3 * A53_MAX_CC_COUNT) > + return AVERROR(EINVAL); > + > + ret = av_buffer_realloc(&s->a53_buf_ref, new_size); > + if (ret >= 0) { > + uint8_t field, cc1, cc2; > + uint8_t *cap = s->a53_buf_ref->data; > + > + memset(s->a53_buf_ref->data + old_size, 0, cc_count * 3); > + for (unsigned i = 0; i < cc_count && get_bits_left(&gb) >= > 26; i++) { > + skip_bits(&gb, 2); // priority > + field = get_bits(&gb, 2); > + skip_bits(&gb, 5); // line_offset > + cc1 = get_bits(&gb, 8); > + cc2 = get_bits(&gb, 8); > + skip_bits(&gb, 1); // marker > + > + if (!field) { // forbidden > + cap[0] = cap[1] = cap[2] = 0x00; > + } else { > + field = (field == 2 ? 1 : 0); > + ////if (!s1->mpeg_enc_ctx.top_field_first) field = > !field; > + cap[0] = 0x04 | field; > + cap[1] = ff_reverse[cc1]; > + cap[2] = ff_reverse[cc2]; > + } > + cap += 3; > + } > + } > + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS; > + } > + return 1; > + } else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01 > && p[3] == 0xf8) { > + int cc_count = 0; > + int i, ret; > + // There is a caption count field in the data, but it is often > + // incorrect. So count the number of captions present. > + for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6) > + cc_count++; > + // Transform the DVD format into A53 Part 4 format > + if (cc_count > 0) { > + int old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0; > + uint64_t new_size = (old_size + cc_count > + * UINT64_C(6)); > + if (new_size > 3*A53_MAX_CC_COUNT) > + return AVERROR(EINVAL); > + > + ret = av_buffer_realloc(&s->a53_buf_ref, new_size); > + if (ret >= 0) { > + uint8_t field1 = !!(p[4] & 0x80); > + uint8_t *cap = s->a53_buf_ref->data; > + p += 5; > + for (i = 0; i < cc_count; i++) { > + cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd; > + cap[1] = p[1]; > + cap[2] = p[2]; > + cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd; > + cap[4] = p[4]; > + cap[5] = p[5]; > + cap += 6; > + p += 6; > + } > + } > + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS; > + } > + return 1; > + } > + return 0; > +} > + > +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* > out) > +{ > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; > + mfxU64 ts; > + int ret; > + > + while (1) { > + int start; > + > + memset(payload.Data, 0, payload.BufSize); > + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), > payload.BufSize); > + return 0; > + } > + if (ret != MFX_ERR_NONE) > + return ret; > + > + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8) > + break; > + > + start = find_start_offset(payload.Data); > + > + start++; > + > + mpeg_decode_a53_cc(avctx, q, &payload.Data[start], > (int)((payload.NumBit + 7) / 8) - start); > + > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d start %d > -> %.s\n", payload.Type, payload.NumBit, start, (char > *)(&payload.Data[start])); > + } > + > + if (!out) > + return 0; > + > + if (q->a53_buf_ref) { > + > + AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, > AV_FRAME_DATA_A53_CC, q->a53_buf_ref); > + if (!sd) > + av_buffer_unref(&q->a53_buf_ref); > + q->a53_buf_ref = NULL; > + } > + > + return 0; > +} > + > static int qsv_decode(AVCodecContext *avctx, QSVContext *q, > AVFrame *frame, int *got_frame, > const AVPacket *avpkt) > @@ -664,6 +966,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext > *q, > insurf, &outsurf, sync); > if (ret == MFX_WRN_DEVICE_BUSY) > av_usleep(500); > + else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO) > + parse_sei_mpeg12(avctx, q, NULL); > > } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE); > > @@ -705,6 +1009,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext > *q, > return AVERROR_BUG; > } > > + switch (avctx->codec_id) { > + case AV_CODEC_ID_MPEG2VIDEO: > + ret = parse_sei_mpeg12(avctx, q, out_frame->frame); > + break; > + case AV_CODEC_ID_H264: > + ret = parse_sei_h264(avctx, q, out_frame->frame); > + break; > + case AV_CODEC_ID_HEVC: > + ret = parse_sei_hevc(avctx, q, out_frame); > + break; > + default: > + ret = 0; > + } > + > + if (ret < 0) > + av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret); > + > out_frame->queued += 1; > > aframe = (QSVAsyncFrame){ sync, out_frame };
> -----Original Message----- > From: Xiang, Haihao <haihao.xiang@intel.com> > Sent: Monday, November 21, 2022 3:45 AM > To: ffmpeg-devel@ffmpeg.org > Cc: softworkz@hotmail.com; kierank@obe.tv; haihao.xiang-at- > intel.com@ffmpeg.org; andreas.rheinhardt@outlook.com > Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement > SEI parsing for QSV decoders > > On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote: > > From: softworkz <softworkz@hotmail.com> > > > > Signed-off-by: softworkz <softworkz@hotmail.com> > > --- > > libavcodec/Makefile | 2 +- > > libavcodec/qsvdec.c | 321 > ++++++++++++++++++++++++++++++++++++++++++++ > > 2 files changed, 322 insertions(+), 1 deletion(-) > > > > diff --git a/libavcodec/Makefile b/libavcodec/Makefile > > index 90c7f113a3..cbddbb0ace 100644 > > --- a/libavcodec/Makefile > > +++ b/libavcodec/Makefile > > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += > mss34dsp.o > > OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o > > OBJS-$(CONFIG_QPELDSP) += qpeldsp.o > > OBJS-$(CONFIG_QSV) += qsv.o > > -OBJS-$(CONFIG_QSVDEC) += qsvdec.o > > +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o > hevc_sei.o > > OBJS-$(CONFIG_QSVENC) += qsvenc.o > > OBJS-$(CONFIG_RANGECODER) += rangecoder.o > > OBJS-$(CONFIG_RDFT) += rdft.o > > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c > > index 73405b5747..467a248224 100644 > > --- a/libavcodec/qsvdec.c > > +++ b/libavcodec/qsvdec.c > > @@ -41,6 +41,7 @@ > > #include "libavutil/time.h" > > #include "libavutil/imgutils.h" > > #include "libavutil/film_grain_params.h" > > +#include <libavutil/reverse.h> > > > > #include "avcodec.h" > > #include "codec_internal.h" > > @@ -49,6 +50,9 @@ > > #include "hwconfig.h" > > #include "qsv.h" > > #include "qsv_internal.h" > > +#include "h264_sei.h" > > +#include "hevc_ps.h" > > +#include "hevc_sei.h" > > > > #if QSV_ONEVPL > > #include <mfxdispatcher.h> > > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 }; > > AV_NOPTS_VALUE : pts_tb.num ? \ > > av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts) > > > > +#define PAYLOAD_BUFFER_SIZE 65535 > > + > > typedef struct QSVAsyncFrame { > > mfxSyncPoint *sync; > > QSVFrame *frame; > > @@ -107,6 +113,9 @@ typedef struct QSVContext { > > > > mfxExtBuffer **ext_buffers; > > int nb_ext_buffers; > > + > > + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE]; > > + AVBufferRef *a53_buf_ref; > > } QSVContext; > > > > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { > > @@ -628,6 +637,299 @@ static int > qsv_export_film_grain(AVCodecContext *avctx, > > mfxExtAV1FilmGrainParam > > } > > #endif > > > > +static int find_start_offset(mfxU8 data[4]) > > +{ > > + if (data[0] == 0 && data[1] == 0 && data[2] == 1) > > + return 3; > > + > > + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == > 1) > > + return 4; > > + > > + return 0; > > +} > > + > > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, > AVFrame* out) > > +{ > > + H264SEIContext sei = { 0 }; > > + GetBitContext gb = { 0 }; > > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], > .BufSize = > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; > > + mfxU64 ts; > > + int ret; > > + > > + while (1) { > > + int start; > > + memset(payload.Data, 0, payload.BufSize); > > + > > + ret = MFXVideoDECODE_GetPayload(q->session, &ts, > &payload); > > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { > > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient > buffer on > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q- > >payload_buffer), > > payload.BufSize); > > + return 0; > > + } > > + if (ret != MFX_ERR_NONE) > > + return ret; > > + > > + if (payload.NumBit == 0 || payload.NumBit >= > payload.BufSize * 8) > > + break; > > + > > + start = find_start_offset(payload.Data); > > + > > + switch (payload.Type) { > > + case SEI_TYPE_BUFFERING_PERIOD: > > + case SEI_TYPE_PIC_TIMING: > > + continue; > > + } > > + > > + if (init_get_bits(&gb, &payload.Data[start], > payload.NumBit - start * > > 8) < 0) > > + av_log(avctx, AV_LOG_ERROR, "Error initializing > bitstream reader > > SEI type: %d Numbits %d error: %d\n", payload.Type, > payload.NumBit, ret); > > + else { > > + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx); > > + > > + if (ret < 0) > > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI > type: > > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > > + else > > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d > Numbits > > %d\n", payload.Type, payload.NumBit); > > + } > > + } > > + > > + if (out) > > + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, > 0); > > + > > + return 0; > > +} > > + > > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, > QSVFrame* > > out) > > +{ > > + HEVCSEI sei = { 0 }; > > + HEVCParamSets ps = { 0 }; > > + GetBitContext gb = { 0 }; > > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], > .BufSize = > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; > > + mfxFrameSurface1 *surface = &out->surface; > > + mfxU64 ts; > > + int ret, has_logged = 0; > > + > > + while (1) { > > + int start; > > + memset(payload.Data, 0, payload.BufSize); > > + > > + ret = MFXVideoDECODE_GetPayload(q->session, &ts, > &payload); > > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { > > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient > buffer on > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q- > >payload_buffer), > > payload.BufSize); > > + return 0; > > + } > > + if (ret != MFX_ERR_NONE) > > + return ret; > > + > > + if (payload.NumBit == 0 || payload.NumBit >= > payload.BufSize * 8) > > + break; > > + > > + if (!has_logged) { > > + has_logged = 1; > > + av_log(avctx, AV_LOG_VERBOSE, "----------------------- > --------- > > ---------\n"); > > + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - > payload > > timestamp: %llu - surface timestamp: %llu\n", ts, surface- > >Data.TimeStamp); > > + } > > + > > + if (ts != surface->Data.TimeStamp) { > > + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp > (%llu) does > > not match surface timestamp: (%llu)\n", ts, surface- > >Data.TimeStamp); > > + } > > + > > + start = find_start_offset(payload.Data); > > + > > + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d > Numbits > > %3d Start: %d\n", payload.Type, payload.NumBit, start); > > + > > + switch (payload.Type) { > > + case SEI_TYPE_BUFFERING_PERIOD: > > + case SEI_TYPE_PIC_TIMING: > > + continue; > > + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME: > > + // There seems to be a bug in MSDK > > + payload.NumBit -= 8; > > + > > + break; > > + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO: > > + // There seems to be a bug in MSDK > > + payload.NumBit = 48; > > + > > + break; > > + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35: > > + // There seems to be a bug in MSDK > > + if (payload.NumBit == 552) > > + payload.NumBit = 528; > > + break; > > + } > > + > > + if (init_get_bits(&gb, &payload.Data[start], > payload.NumBit - start * > > 8) < 0) > > + av_log(avctx, AV_LOG_ERROR, "Error initializing > bitstream reader > > SEI type: %d Numbits %d error: %d\n", payload.Type, > payload.NumBit, ret); > > + else { > > + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, > > HEVC_NAL_SEI_PREFIX); > > + > > + if (ret < 0) > > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI > type: > > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > > + else > > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d > Numbits > > %d\n", payload.Type, payload.NumBit); > > + } > > + } > > + > > + if (has_logged) { > > + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n"); > > + } > > + > > + if (out && out->frame) > > + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, > avctx- > > >framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps- > >bit_depth_chroma); > > I got segfault when trying your patchset, > > Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault. > 0x00007ffff67c0497 in parse_sei_hevc > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 > 777 return ff_hevc_set_sei_to_frame(avctx, &sei, out- > >frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps- > >bit_depth_chroma); > (gdb) bt > #0 0x00007ffff67c0497 in parse_sei_hevc > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 > #1 0x00007ffff67c1afe in qsv_decode > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, > frame=frame@entry=0x5555556df740, > got_frame=got_frame@entry=0x7fffffffd6bc, > avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020 > BTW the SDK provides support for hevc HDR metadata, we needn't parse > SEI payload > in qsvdec and may get the corresponding info from the SDK, see > https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html I know. I was the one who had requested this to be added to MSDK :-) But it's just one small part of SEI information, it's limited to the latest MSDK versions and I'm not sure whether it's working as reliably as this implementation. This would need to be tested. You should still have access to the repo with the test files for demoing the offset problems (which my patchset is working around). But it's also about dynamic HDR data, dovi data, etc. - which MSDK doesn't provide, so this single bit of SEI data doesn't help much. Best regards, softworkz
On Mon, 2022-11-21 at 15:58 +0000, Soft Works wrote: > > -----Original Message----- > > From: Xiang, Haihao <haihao.xiang@intel.com> > > Sent: Monday, November 21, 2022 3:45 AM > > To: ffmpeg-devel@ffmpeg.org > > Cc: softworkz@hotmail.com; kierank@obe.tv; haihao.xiang-at- > > intel.com@ffmpeg.org; andreas.rheinhardt@outlook.com > > Subject: Re: [FFmpeg-devel] [PATCH v6 3/3] avcodec/qsvdec: Implement > > SEI parsing for QSV decoders > > > > On Tue, 2022-10-25 at 04:03 +0000, softworkz wrote: > > > From: softworkz <softworkz@hotmail.com> > > > > > > Signed-off-by: softworkz <softworkz@hotmail.com> > > > --- > > > libavcodec/Makefile | 2 +- > > > libavcodec/qsvdec.c | 321 > > ++++++++++++++++++++++++++++++++++++++++++++ > > > 2 files changed, 322 insertions(+), 1 deletion(-) > > > > > > diff --git a/libavcodec/Makefile b/libavcodec/Makefile > > > index 90c7f113a3..cbddbb0ace 100644 > > > --- a/libavcodec/Makefile > > > +++ b/libavcodec/Makefile > > > @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += > > mss34dsp.o > > > OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o > > > OBJS-$(CONFIG_QPELDSP) += qpeldsp.o > > > OBJS-$(CONFIG_QSV) += qsv.o > > > -OBJS-$(CONFIG_QSVDEC) += qsvdec.o > > > +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o > > hevc_sei.o > > > OBJS-$(CONFIG_QSVENC) += qsvenc.o > > > OBJS-$(CONFIG_RANGECODER) += rangecoder.o > > > OBJS-$(CONFIG_RDFT) += rdft.o > > > diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c > > > index 73405b5747..467a248224 100644 > > > --- a/libavcodec/qsvdec.c > > > +++ b/libavcodec/qsvdec.c > > > @@ -41,6 +41,7 @@ > > > #include "libavutil/time.h" > > > #include "libavutil/imgutils.h" > > > #include "libavutil/film_grain_params.h" > > > +#include <libavutil/reverse.h> > > > > > > #include "avcodec.h" > > > #include "codec_internal.h" > > > @@ -49,6 +50,9 @@ > > > #include "hwconfig.h" > > > #include "qsv.h" > > > #include "qsv_internal.h" > > > +#include "h264_sei.h" > > > +#include "hevc_ps.h" > > > +#include "hevc_sei.h" > > > > > > #if QSV_ONEVPL > > > #include <mfxdispatcher.h> > > > @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 }; > > > AV_NOPTS_VALUE : pts_tb.num ? \ > > > av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts) > > > > > > +#define PAYLOAD_BUFFER_SIZE 65535 > > > + > > > typedef struct QSVAsyncFrame { > > > mfxSyncPoint *sync; > > > QSVFrame *frame; > > > @@ -107,6 +113,9 @@ typedef struct QSVContext { > > > > > > mfxExtBuffer **ext_buffers; > > > int nb_ext_buffers; > > > + > > > + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE]; > > > + AVBufferRef *a53_buf_ref; > > > } QSVContext; > > > > > > static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { > > > @@ -628,6 +637,299 @@ static int > > qsv_export_film_grain(AVCodecContext *avctx, > > > mfxExtAV1FilmGrainParam > > > } > > > #endif > > > > > > +static int find_start_offset(mfxU8 data[4]) > > > +{ > > > + if (data[0] == 0 && data[1] == 0 && data[2] == 1) > > > + return 3; > > > + > > > + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == > > 1) > > > + return 4; > > > + > > > + return 0; > > > +} > > > + > > > +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, > > AVFrame* out) > > > +{ > > > + H264SEIContext sei = { 0 }; > > > + GetBitContext gb = { 0 }; > > > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], > > .BufSize = > > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; > > > + mfxU64 ts; > > > + int ret; > > > + > > > + while (1) { > > > + int start; > > > + memset(payload.Data, 0, payload.BufSize); > > > + > > > + ret = MFXVideoDECODE_GetPayload(q->session, &ts, > > &payload); > > > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { > > > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient > > buffer on > > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q- > > > payload_buffer), > > > payload.BufSize); > > > + return 0; > > > + } > > > + if (ret != MFX_ERR_NONE) > > > + return ret; > > > + > > > + if (payload.NumBit == 0 || payload.NumBit >= > > payload.BufSize * 8) > > > + break; > > > + > > > + start = find_start_offset(payload.Data); > > > + > > > + switch (payload.Type) { > > > + case SEI_TYPE_BUFFERING_PERIOD: > > > + case SEI_TYPE_PIC_TIMING: > > > + continue; > > > + } > > > + > > > + if (init_get_bits(&gb, &payload.Data[start], > > payload.NumBit - start * > > > 8) < 0) > > > + av_log(avctx, AV_LOG_ERROR, "Error initializing > > bitstream reader > > > SEI type: %d Numbits %d error: %d\n", payload.Type, > > payload.NumBit, ret); > > > + else { > > > + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx); > > > + > > > + if (ret < 0) > > > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI > > type: > > > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > > > + else > > > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d > > Numbits > > > %d\n", payload.Type, payload.NumBit); > > > + } > > > + } > > > + > > > + if (out) > > > + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, > > 0); > > > + > > > + return 0; > > > +} > > > + > > > +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, > > QSVFrame* > > > out) > > > +{ > > > + HEVCSEI sei = { 0 }; > > > + HEVCParamSets ps = { 0 }; > > > + GetBitContext gb = { 0 }; > > > + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], > > .BufSize = > > > sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; > > > + mfxFrameSurface1 *surface = &out->surface; > > > + mfxU64 ts; > > > + int ret, has_logged = 0; > > > + > > > + while (1) { > > > + int start; > > > + memset(payload.Data, 0, payload.BufSize); > > > + > > > + ret = MFXVideoDECODE_GetPayload(q->session, &ts, > > &payload); > > > + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { > > > + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient > > buffer on > > > GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q- > > > payload_buffer), > > > payload.BufSize); > > > + return 0; > > > + } > > > + if (ret != MFX_ERR_NONE) > > > + return ret; > > > + > > > + if (payload.NumBit == 0 || payload.NumBit >= > > payload.BufSize * 8) > > > + break; > > > + > > > + if (!has_logged) { > > > + has_logged = 1; > > > + av_log(avctx, AV_LOG_VERBOSE, "----------------------- > > --------- > > > ---------\n"); > > > + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - > > payload > > > timestamp: %llu - surface timestamp: %llu\n", ts, surface- > > > Data.TimeStamp); > > > + } > > > + > > > + if (ts != surface->Data.TimeStamp) { > > > + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp > > (%llu) does > > > not match surface timestamp: (%llu)\n", ts, surface- > > > Data.TimeStamp); > > > + } > > > + > > > + start = find_start_offset(payload.Data); > > > + > > > + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d > > Numbits > > > %3d Start: %d\n", payload.Type, payload.NumBit, start); > > > + > > > + switch (payload.Type) { > > > + case SEI_TYPE_BUFFERING_PERIOD: > > > + case SEI_TYPE_PIC_TIMING: > > > + continue; > > > + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME: > > > + // There seems to be a bug in MSDK > > > + payload.NumBit -= 8; > > > + > > > + break; > > > + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO: > > > + // There seems to be a bug in MSDK > > > + payload.NumBit = 48; > > > + > > > + break; > > > + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35: > > > + // There seems to be a bug in MSDK > > > + if (payload.NumBit == 552) > > > + payload.NumBit = 528; > > > + break; > > > + } > > > + > > > + if (init_get_bits(&gb, &payload.Data[start], > > payload.NumBit - start * > > > 8) < 0) > > > + av_log(avctx, AV_LOG_ERROR, "Error initializing > > bitstream reader > > > SEI type: %d Numbits %d error: %d\n", payload.Type, > > payload.NumBit, ret); > > > + else { > > > + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, > > > HEVC_NAL_SEI_PREFIX); > > > + > > > + if (ret < 0) > > > + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI > > type: > > > %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); > > > + else > > > + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d > > Numbits > > > %d\n", payload.Type, payload.NumBit); > > > + } > > > + } > > > + > > > + if (has_logged) { > > > + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n"); > > > + } > > > + > > > + if (out && out->frame) > > > + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, > > avctx- > > > > framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps- > > > bit_depth_chroma); > > > > I got segfault when trying your patchset, > > > > Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault. > > 0x00007ffff67c0497 in parse_sei_hevc > > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, > > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 > > 777 return ff_hevc_set_sei_to_frame(avctx, &sei, out- > > > frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps- > > > bit_depth_chroma); > > (gdb) bt > > #0 0x00007ffff67c0497 in parse_sei_hevc > > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, > > out=out@entry=0x5555559b6f80) at libavcodec/qsvdec.c:777 > > #1 0x00007ffff67c1afe in qsv_decode > > (avctx=avctx@entry=0x5555555e4280, q=q@entry=0x555555625288, > > frame=frame@entry=0x5555556df740, > > got_frame=got_frame@entry=0x7fffffffd6bc, > > avpkt=avpkt@entry=0x555555635398) at libavcodec/qsvdec.c:1020 > > BTW the SDK provides support for hevc HDR metadata, we needn't parse > > SEI payload > > in qsvdec and may get the corresponding info from the SDK, see > > https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html > > I know. I was the one who had requested this to be added to MSDK :-) > > But it's just one small part of SEI information, it's limited to > the latest MSDK versions and I'm not sure whether it's working > as reliably as this implementation. This would need to be tested. > > You should still have access to the repo with the test files for > demoing the offset problems (which my patchset is working around). I worked out a patchset (https://github.com/intel-media-ci/ffmpeg/pull/518), including https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html an d others. I may use your command (with some changes) to convert all HDR clips in your repo to SDR clips except one clip (w/wo my changes, there is a GPU hang issue with this clip). We may only do a small update to support AV1 HDR in the future if applying https://ffmpeg.org/pipermail/ffmpeg-devel/2022-November/304142.html. Note the command below doesn't work with all test files in your repo with your patchset v6. $ ffmpeg -hwaccel qsv -c:v hevc_qsv -i input.mp4 -f null - Thanks Haihao > > But it's also about dynamic HDR data, dovi data, etc. - which > MSDK doesn't provide, so this single bit of SEI data doesn't > help much. > > Best regards, > softworkz > > >
diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 90c7f113a3..cbddbb0ace 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP) += mss34dsp.o OBJS-$(CONFIG_PIXBLOCKDSP) += pixblockdsp.o OBJS-$(CONFIG_QPELDSP) += qpeldsp.o OBJS-$(CONFIG_QSV) += qsv.o -OBJS-$(CONFIG_QSVDEC) += qsvdec.o +OBJS-$(CONFIG_QSVDEC) += qsvdec.o h264_sei.o hevc_sei.o OBJS-$(CONFIG_QSVENC) += qsvenc.o OBJS-$(CONFIG_RANGECODER) += rangecoder.o OBJS-$(CONFIG_RDFT) += rdft.o diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c index 73405b5747..467a248224 100644 --- a/libavcodec/qsvdec.c +++ b/libavcodec/qsvdec.c @@ -41,6 +41,7 @@ #include "libavutil/time.h" #include "libavutil/imgutils.h" #include "libavutil/film_grain_params.h" +#include <libavutil/reverse.h> #include "avcodec.h" #include "codec_internal.h" @@ -49,6 +50,9 @@ #include "hwconfig.h" #include "qsv.h" #include "qsv_internal.h" +#include "h264_sei.h" +#include "hevc_ps.h" +#include "hevc_sei.h" #if QSV_ONEVPL #include <mfxdispatcher.h> @@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 }; AV_NOPTS_VALUE : pts_tb.num ? \ av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts) +#define PAYLOAD_BUFFER_SIZE 65535 + typedef struct QSVAsyncFrame { mfxSyncPoint *sync; QSVFrame *frame; @@ -107,6 +113,9 @@ typedef struct QSVContext { mfxExtBuffer **ext_buffers; int nb_ext_buffers; + + mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE]; + AVBufferRef *a53_buf_ref; } QSVContext; static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { @@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam } #endif +static int find_start_offset(mfxU8 data[4]) +{ + if (data[0] == 0 && data[1] == 0 && data[2] == 1) + return 3; + + if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) + return 4; + + return 0; +} + +static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out) +{ + H264SEIContext sei = { 0 }; + GetBitContext gb = { 0 }; + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; + mfxU64 ts; + int ret; + + while (1) { + int start; + memset(payload.Data, 0, payload.BufSize); + + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize); + return 0; + } + if (ret != MFX_ERR_NONE) + return ret; + + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8) + break; + + start = find_start_offset(payload.Data); + + switch (payload.Type) { + case SEI_TYPE_BUFFERING_PERIOD: + case SEI_TYPE_PIC_TIMING: + continue; + } + + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0) + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); + else { + ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx); + + if (ret < 0) + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); + else + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit); + } + } + + if (out) + return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0); + + return 0; +} + +static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* out) +{ + HEVCSEI sei = { 0 }; + HEVCParamSets ps = { 0 }; + GetBitContext gb = { 0 }; + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; + mfxFrameSurface1 *surface = &out->surface; + mfxU64 ts; + int ret, has_logged = 0; + + while (1) { + int start; + memset(payload.Data, 0, payload.BufSize); + + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize); + return 0; + } + if (ret != MFX_ERR_NONE) + return ret; + + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8) + break; + + if (!has_logged) { + has_logged = 1; + av_log(avctx, AV_LOG_VERBOSE, "-----------------------------------------\n"); + av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp); + } + + if (ts != surface->Data.TimeStamp) { + av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp); + } + + start = find_start_offset(payload.Data); + + av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d Numbits %3d Start: %d\n", payload.Type, payload.NumBit, start); + + switch (payload.Type) { + case SEI_TYPE_BUFFERING_PERIOD: + case SEI_TYPE_PIC_TIMING: + continue; + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME: + // There seems to be a bug in MSDK + payload.NumBit -= 8; + + break; + case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO: + // There seems to be a bug in MSDK + payload.NumBit = 48; + + break; + case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35: + // There seems to be a bug in MSDK + if (payload.NumBit == 552) + payload.NumBit = 528; + break; + } + + if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0) + av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); + else { + ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX); + + if (ret < 0) + av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d Numbits %d error: %d\n", payload.Type, payload.NumBit, ret); + else + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d\n", payload.Type, payload.NumBit); + } + } + + if (has_logged) { + av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n"); + } + + if (out && out->frame) + return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma); + + return 0; +} + +#define A53_MAX_CC_COUNT 2000 + +static int mpeg_decode_a53_cc(AVCodecContext *avctx, QSVContext *s, + const uint8_t *p, int buf_size) +{ + if (buf_size >= 6 && + p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' && + p[4] == 3 && (p[5] & 0x40)) { + /* extract A53 Part 4 CC data */ + unsigned cc_count = p[5] & 0x1f; + if (cc_count > 0 && buf_size >= 7 + cc_count * 3) { + const uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0; + const uint64_t new_size = (old_size + cc_count + * UINT64_C(3)); + int ret; + + if (new_size > 3*A53_MAX_CC_COUNT) + return AVERROR(EINVAL); + + ret = av_buffer_realloc(&s->a53_buf_ref, new_size); + if (ret >= 0) + memcpy(s->a53_buf_ref->data + old_size, p + 7, cc_count * UINT64_C(3)); + + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS; + } + return 1; + } else if (buf_size >= 2 && p[0] == 0x03 && (p[1]&0x7f) == 0x01) { + /* extract SCTE-20 CC data */ + GetBitContext gb; + unsigned cc_count = 0; + int ret; + + init_get_bits8(&gb, p + 2, buf_size - 2); + cc_count = get_bits(&gb, 5); + if (cc_count > 0) { + uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0; + uint64_t new_size = (old_size + cc_count * UINT64_C(3)); + if (new_size > 3 * A53_MAX_CC_COUNT) + return AVERROR(EINVAL); + + ret = av_buffer_realloc(&s->a53_buf_ref, new_size); + if (ret >= 0) { + uint8_t field, cc1, cc2; + uint8_t *cap = s->a53_buf_ref->data; + + memset(s->a53_buf_ref->data + old_size, 0, cc_count * 3); + for (unsigned i = 0; i < cc_count && get_bits_left(&gb) >= 26; i++) { + skip_bits(&gb, 2); // priority + field = get_bits(&gb, 2); + skip_bits(&gb, 5); // line_offset + cc1 = get_bits(&gb, 8); + cc2 = get_bits(&gb, 8); + skip_bits(&gb, 1); // marker + + if (!field) { // forbidden + cap[0] = cap[1] = cap[2] = 0x00; + } else { + field = (field == 2 ? 1 : 0); + ////if (!s1->mpeg_enc_ctx.top_field_first) field = !field; + cap[0] = 0x04 | field; + cap[1] = ff_reverse[cc1]; + cap[2] = ff_reverse[cc2]; + } + cap += 3; + } + } + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS; + } + return 1; + } else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01 && p[3] == 0xf8) { + int cc_count = 0; + int i, ret; + // There is a caption count field in the data, but it is often + // incorrect. So count the number of captions present. + for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6) + cc_count++; + // Transform the DVD format into A53 Part 4 format + if (cc_count > 0) { + int old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0; + uint64_t new_size = (old_size + cc_count + * UINT64_C(6)); + if (new_size > 3*A53_MAX_CC_COUNT) + return AVERROR(EINVAL); + + ret = av_buffer_realloc(&s->a53_buf_ref, new_size); + if (ret >= 0) { + uint8_t field1 = !!(p[4] & 0x80); + uint8_t *cap = s->a53_buf_ref->data; + p += 5; + for (i = 0; i < cc_count; i++) { + cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd; + cap[1] = p[1]; + cap[2] = p[2]; + cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd; + cap[4] = p[4]; + cap[5] = p[5]; + cap += 6; + p += 6; + } + } + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS; + } + return 1; + } + return 0; +} + +static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* out) +{ + mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE }; + mfxU64 ts; + int ret; + + while (1) { + int start; + + memset(payload.Data, 0, payload.BufSize); + ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload); + if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) { + av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize); + return 0; + } + if (ret != MFX_ERR_NONE) + return ret; + + if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8) + break; + + start = find_start_offset(payload.Data); + + start++; + + mpeg_decode_a53_cc(avctx, q, &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start); + + av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d Numbits %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char *)(&payload.Data[start])); + } + + if (!out) + return 0; + + if (q->a53_buf_ref) { + + AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, q->a53_buf_ref); + if (!sd) + av_buffer_unref(&q->a53_buf_ref); + q->a53_buf_ref = NULL; + } + + return 0; +} + static int qsv_decode(AVCodecContext *avctx, QSVContext *q, AVFrame *frame, int *got_frame, const AVPacket *avpkt) @@ -664,6 +966,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q, insurf, &outsurf, sync); if (ret == MFX_WRN_DEVICE_BUSY) av_usleep(500); + else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO) + parse_sei_mpeg12(avctx, q, NULL); } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE); @@ -705,6 +1009,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q, return AVERROR_BUG; } + switch (avctx->codec_id) { + case AV_CODEC_ID_MPEG2VIDEO: + ret = parse_sei_mpeg12(avctx, q, out_frame->frame); + break; + case AV_CODEC_ID_H264: + ret = parse_sei_h264(avctx, q, out_frame->frame); + break; + case AV_CODEC_ID_HEVC: + ret = parse_sei_hevc(avctx, q, out_frame); + break; + default: + ret = 0; + } + + if (ret < 0) + av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret); + out_frame->queued += 1; aframe = (QSVAsyncFrame){ sync, out_frame };