diff mbox series

[FFmpeg-devel,13/18] h264_sei: parse the picture timing SEIs correctly

Message ID 20200313102850.23913-13-anton@khirnov.net
State Accepted
Headers show
Series [FFmpeg-devel,01/18] mpeg4videodec: do not copy a range of fields at once
Related show

Checks

Context Check Description
andriy/ffmpeg-patchwork pending
andriy/ffmpeg-patchwork warning Failed to apply patch

Commit Message

Anton Khirnov March 13, 2020, 10:28 a.m. UTC
Those SEIs refer to the currently active SPS. However, since the SEI
NALUs precede the coded picture data in the bitstream, the active SPS is
in general not known when we are decoding the SEI.

Therefore, store the content of the picture timing SEIs and actually
parse it when the active SPS is known.
---
 libavcodec/h264_parser.c |  9 +++++
 libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
 libavcodec/h264_sei.h    | 11 +++++
 libavcodec/h264_slice.c  | 10 +++++
 4 files changed, 82 insertions(+), 34 deletions(-)

Comments

James Almer March 13, 2020, 3:56 p.m. UTC | #1
On 3/13/2020 7:28 AM, Anton Khirnov wrote:
> Those SEIs refer to the currently active SPS. However, since the SEI
> NALUs precede the coded picture data in the bitstream, the active SPS is
> in general not known when we are decoding the SEI.
> 
> Therefore, store the content of the picture timing SEIs and actually
> parse it when the active SPS is known.
> ---
>  libavcodec/h264_parser.c |  9 +++++
>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
>  libavcodec/h264_sei.h    | 11 +++++
>  libavcodec/h264_slice.c  | 10 +++++
>  4 files changed, 82 insertions(+), 34 deletions(-)
> 

[...]

> +static int decode_picture_timing(H264SEIPictureTiming *h, GetBitContext *gb,
> +                                 void *logctx)
> +{
> +    PutBitContext pb;
> +    int n;
> +
> +    if (get_bits_left(gb) > sizeof(h->payload) * 8) {
> +        av_log(logctx, AV_LOG_ERROR, "Picture timing SEI payload too large\n");
> +        return AVERROR_INVALIDDATA;
> +    }
> +
> +    init_put_bits(&pb, h->payload, sizeof(h->payload));
> +
> +    while (get_bits_left(gb) >= 32)
> +        put_bits(&pb, 32, get_bits_long(gb, 32));
> +
> +    n = get_bits_left(gb);
> +    if (n)
> +        put_bits(&pb, n, get_bits_long(gb, n));

I think you can do something like avpriv_copy_bits(&pb, gb->buffer,
get_bits_left(gb)) instead.
No need to skip the bits in gb afterwards since the GetBitContext struct
is exclusive for this SEI starting with the previous patch.

> +
> +    h->payload_size_bits = put_bits_count(&pb);
> +    flush_put_bits(&pb);
> +
>      h->present = 1;
>      return 0;
>  }
Anton Khirnov March 16, 2020, 2:59 p.m. UTC | #2
Quoting James Almer (2020-03-13 16:56:12)
> On 3/13/2020 7:28 AM, Anton Khirnov wrote:
> > Those SEIs refer to the currently active SPS. However, since the SEI
> > NALUs precede the coded picture data in the bitstream, the active SPS is
> > in general not known when we are decoding the SEI.
> > 
> > Therefore, store the content of the picture timing SEIs and actually
> > parse it when the active SPS is known.
> > ---
> >  libavcodec/h264_parser.c |  9 +++++
> >  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
> >  libavcodec/h264_sei.h    | 11 +++++
> >  libavcodec/h264_slice.c  | 10 +++++
> >  4 files changed, 82 insertions(+), 34 deletions(-)
> > 
> 
> [...]
> 
> > +static int decode_picture_timing(H264SEIPictureTiming *h, GetBitContext *gb,
> > +                                 void *logctx)
> > +{
> > +    PutBitContext pb;
> > +    int n;
> > +
> > +    if (get_bits_left(gb) > sizeof(h->payload) * 8) {
> > +        av_log(logctx, AV_LOG_ERROR, "Picture timing SEI payload too large\n");
> > +        return AVERROR_INVALIDDATA;
> > +    }
> > +
> > +    init_put_bits(&pb, h->payload, sizeof(h->payload));
> > +
> > +    while (get_bits_left(gb) >= 32)
> > +        put_bits(&pb, 32, get_bits_long(gb, 32));
> > +
> > +    n = get_bits_left(gb);
> > +    if (n)
> > +        put_bits(&pb, n, get_bits_long(gb, n));
> 
> I think you can do something like avpriv_copy_bits(&pb, gb->buffer,
> get_bits_left(gb)) instead.

Right, I wasn't sure if we want to depend on payload start being
byte-aligned. I suppose it has to be true in valid streams so it should
be ok.

> No need to skip the bits in gb afterwards since the GetBitContext struct
> is exclusive for this SEI starting with the previous patch.

Not sure which skip you mean here.
James Almer March 16, 2020, 3:05 p.m. UTC | #3
On 3/16/2020 11:59 AM, Anton Khirnov wrote:
> Quoting James Almer (2020-03-13 16:56:12)
>> On 3/13/2020 7:28 AM, Anton Khirnov wrote:
>>> Those SEIs refer to the currently active SPS. However, since the SEI
>>> NALUs precede the coded picture data in the bitstream, the active SPS is
>>> in general not known when we are decoding the SEI.
>>>
>>> Therefore, store the content of the picture timing SEIs and actually
>>> parse it when the active SPS is known.
>>> ---
>>>  libavcodec/h264_parser.c |  9 +++++
>>>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
>>>  libavcodec/h264_sei.h    | 11 +++++
>>>  libavcodec/h264_slice.c  | 10 +++++
>>>  4 files changed, 82 insertions(+), 34 deletions(-)
>>>
>>
>> [...]
>>
>>> +static int decode_picture_timing(H264SEIPictureTiming *h, GetBitContext *gb,
>>> +                                 void *logctx)
>>> +{
>>> +    PutBitContext pb;
>>> +    int n;
>>> +
>>> +    if (get_bits_left(gb) > sizeof(h->payload) * 8) {
>>> +        av_log(logctx, AV_LOG_ERROR, "Picture timing SEI payload too large\n");
>>> +        return AVERROR_INVALIDDATA;
>>> +    }
>>> +
>>> +    init_put_bits(&pb, h->payload, sizeof(h->payload));
>>> +
>>> +    while (get_bits_left(gb) >= 32)
>>> +        put_bits(&pb, 32, get_bits_long(gb, 32));
>>> +
>>> +    n = get_bits_left(gb);
>>> +    if (n)
>>> +        put_bits(&pb, n, get_bits_long(gb, n));
>>
>> I think you can do something like avpriv_copy_bits(&pb, gb->buffer,
>> get_bits_left(gb)) instead.
> 
> Right, I wasn't sure if we want to depend on payload start being
> byte-aligned. I suppose it has to be true in valid streams so it should
> be ok.
> 
>> No need to skip the bits in gb afterwards since the GetBitContext struct
>> is exclusive for this SEI starting with the previous patch.
> 
> Not sure which skip you mean here.

avpriv_copy_bits() takes an uint8_t* argument as input, and unlike when
doing put_bits(&pb, 32, get_bits_long(gb, 32)), the GetBitContext will
remain unchanged.
If if was required to keep parsing the following SEI message in the same
context you'd need to do skip_bits_long() on it. But that's not needed
now that each individual message uses it's own context.
Michael Niedermayer March 16, 2020, 6:51 p.m. UTC | #4
On Fri, Mar 13, 2020 at 11:28:45AM +0100, Anton Khirnov wrote:
> Those SEIs refer to the currently active SPS. However, since the SEI
> NALUs precede the coded picture data in the bitstream, the active SPS is
> in general not known when we are decoding the SEI.
> 
> Therefore, store the content of the picture timing SEIs and actually
> parse it when the active SPS is known.
> ---
>  libavcodec/h264_parser.c |  9 +++++
>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
>  libavcodec/h264_sei.h    | 11 +++++
>  libavcodec/h264_slice.c  | 10 +++++
>  4 files changed, 82 insertions(+), 34 deletions(-)

This seems to break fate

TEST    cbs-h264-CVSE2_Sony_B
--- ./tests/ref/fate/cbs-h264-CVSE2_Sony_B	2020-03-15 16:08:13.692521532 +0100
+++ tests/data/fate/cbs-h264-CVSE2_Sony_B	2020-03-16 19:50:23.164302663 +0100
@@ -1 +0,0 @@
-ca8bdba497bd2f3b97c50d59692eb537
Test cbs-h264-CVSE2_Sony_B failed. Look at tests/data/fate/cbs-h264-CVSE2_Sony_B.err for details.
tests/Makefile:250: recipe for target 'fate-cbs-h264-CVSE2_Sony_B' failed
make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1

thx

[...]
James Almer March 16, 2020, 7:42 p.m. UTC | #5
On 3/16/2020 3:51 PM, Michael Niedermayer wrote:
> On Fri, Mar 13, 2020 at 11:28:45AM +0100, Anton Khirnov wrote:
>> Those SEIs refer to the currently active SPS. However, since the SEI
>> NALUs precede the coded picture data in the bitstream, the active SPS is
>> in general not known when we are decoding the SEI.
>>
>> Therefore, store the content of the picture timing SEIs and actually
>> parse it when the active SPS is known.
>> ---
>>  libavcodec/h264_parser.c |  9 +++++
>>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
>>  libavcodec/h264_sei.h    | 11 +++++
>>  libavcodec/h264_slice.c  | 10 +++++
>>  4 files changed, 82 insertions(+), 34 deletions(-)
> 
> This seems to break fate
> 
> TEST    cbs-h264-CVSE2_Sony_B
> --- ./tests/ref/fate/cbs-h264-CVSE2_Sony_B	2020-03-15 16:08:13.692521532 +0100
> +++ tests/data/fate/cbs-h264-CVSE2_Sony_B	2020-03-16 19:50:23.164302663 +0100
> @@ -1 +0,0 @@
> -ca8bdba497bd2f3b97c50d59692eb537
> Test cbs-h264-CVSE2_Sony_B failed. Look at tests/data/fate/cbs-h264-CVSE2_Sony_B.err for details.
> tests/Makefile:250: recipe for target 'fate-cbs-h264-CVSE2_Sony_B' failed
> make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1
> 
> thx

I can't reproduce this. Are you using a clean tree?
Anton Khirnov March 16, 2020, 7:50 p.m. UTC | #6
Quoting James Almer (2020-03-16 20:42:31)
> On 3/16/2020 3:51 PM, Michael Niedermayer wrote:
> > On Fri, Mar 13, 2020 at 11:28:45AM +0100, Anton Khirnov wrote:
> >> Those SEIs refer to the currently active SPS. However, since the SEI
> >> NALUs precede the coded picture data in the bitstream, the active SPS is
> >> in general not known when we are decoding the SEI.
> >>
> >> Therefore, store the content of the picture timing SEIs and actually
> >> parse it when the active SPS is known.
> >> ---
> >>  libavcodec/h264_parser.c |  9 +++++
> >>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
> >>  libavcodec/h264_sei.h    | 11 +++++
> >>  libavcodec/h264_slice.c  | 10 +++++
> >>  4 files changed, 82 insertions(+), 34 deletions(-)
> > 
> > This seems to break fate
> > 
> > TEST    cbs-h264-CVSE2_Sony_B
> > --- ./tests/ref/fate/cbs-h264-CVSE2_Sony_B    2020-03-15 16:08:13.692521532 +0100
> > +++ tests/data/fate/cbs-h264-CVSE2_Sony_B     2020-03-16 19:50:23.164302663 +0100
> > @@ -1 +0,0 @@
> > -ca8bdba497bd2f3b97c50d59692eb537
> > Test cbs-h264-CVSE2_Sony_B failed. Look at tests/data/fate/cbs-h264-CVSE2_Sony_B.err for details.
> > tests/Makefile:250: recipe for target 'fate-cbs-h264-CVSE2_Sony_B' failed
> > make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1
> > 
> > thx
> 
> I can't reproduce this. Are you using a clean tree?

Yeah, me neither. I ran FATE on the tree before sending it.
Michael Niedermayer March 17, 2020, 12:13 a.m. UTC | #7
On Mon, Mar 16, 2020 at 04:42:31PM -0300, James Almer wrote:
> On 3/16/2020 3:51 PM, Michael Niedermayer wrote:
> > On Fri, Mar 13, 2020 at 11:28:45AM +0100, Anton Khirnov wrote:
> >> Those SEIs refer to the currently active SPS. However, since the SEI
> >> NALUs precede the coded picture data in the bitstream, the active SPS is
> >> in general not known when we are decoding the SEI.
> >>
> >> Therefore, store the content of the picture timing SEIs and actually
> >> parse it when the active SPS is known.
> >> ---
> >>  libavcodec/h264_parser.c |  9 +++++
> >>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
> >>  libavcodec/h264_sei.h    | 11 +++++
> >>  libavcodec/h264_slice.c  | 10 +++++
> >>  4 files changed, 82 insertions(+), 34 deletions(-)
> > 
> > This seems to break fate
> > 
> > TEST    cbs-h264-CVSE2_Sony_B
> > --- ./tests/ref/fate/cbs-h264-CVSE2_Sony_B	2020-03-15 16:08:13.692521532 +0100
> > +++ tests/data/fate/cbs-h264-CVSE2_Sony_B	2020-03-16 19:50:23.164302663 +0100
> > @@ -1 +0,0 @@
> > -ca8bdba497bd2f3b97c50d59692eb537
> > Test cbs-h264-CVSE2_Sony_B failed. Look at tests/data/fate/cbs-h264-CVSE2_Sony_B.err for details.
> > tests/Makefile:250: recipe for target 'fate-cbs-h264-CVSE2_Sony_B' failed
> > make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1
> > 
> > thx
> 
> I can't reproduce this. Are you using a clean tree?

i had other patches in there, ill retest without tomorrow (too late now)
disregard my comment about there being an issue until i retest

thx


[...]
Michael Niedermayer March 21, 2020, 10:44 p.m. UTC | #8
On Tue, Mar 17, 2020 at 01:13:53AM +0100, Michael Niedermayer wrote:
> On Mon, Mar 16, 2020 at 04:42:31PM -0300, James Almer wrote:
> > On 3/16/2020 3:51 PM, Michael Niedermayer wrote:
> > > On Fri, Mar 13, 2020 at 11:28:45AM +0100, Anton Khirnov wrote:
> > >> Those SEIs refer to the currently active SPS. However, since the SEI
> > >> NALUs precede the coded picture data in the bitstream, the active SPS is
> > >> in general not known when we are decoding the SEI.
> > >>
> > >> Therefore, store the content of the picture timing SEIs and actually
> > >> parse it when the active SPS is known.
> > >> ---
> > >>  libavcodec/h264_parser.c |  9 +++++
> > >>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
> > >>  libavcodec/h264_sei.h    | 11 +++++
> > >>  libavcodec/h264_slice.c  | 10 +++++
> > >>  4 files changed, 82 insertions(+), 34 deletions(-)
> > > 
> > > This seems to break fate
> > > 
> > > TEST    cbs-h264-CVSE2_Sony_B
> > > --- ./tests/ref/fate/cbs-h264-CVSE2_Sony_B	2020-03-15 16:08:13.692521532 +0100
> > > +++ tests/data/fate/cbs-h264-CVSE2_Sony_B	2020-03-16 19:50:23.164302663 +0100
> > > @@ -1 +0,0 @@
> > > -ca8bdba497bd2f3b97c50d59692eb537
> > > Test cbs-h264-CVSE2_Sony_B failed. Look at tests/data/fate/cbs-h264-CVSE2_Sony_B.err for details.
> > > tests/Makefile:250: recipe for target 'fate-cbs-h264-CVSE2_Sony_B' failed
> > > make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1
> > > 
> > > thx
> > 
> > I can't reproduce this. Are you using a clean tree?
> 
> i had other patches in there, ill retest without tomorrow (too late now)
> disregard my comment about there being an issue until i retest

sorry for te delay
the failures persist with patches 1 to this

make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1
make: *** [fate-cbs-h264-sei-1] Error 1
make: *** [fate-h264-timecode] Error 134
make: *** [fate-h264-mixed-nal-coding] Error 134
make: *** [fate-h264-twofields-packet] Error 134
make: *** [fate-h264-attachment-631] Error 134
make: *** [fate-mov-tenc-only-encrypted] Error 134
make: *** [fate-mov-frag-encrypted] Error 134
make: *** [fate-mov-bbi-elst-starts-b] Error 134
make: *** [fate-h264_mp4toannexb_ticket2991] Error 134
make: *** [fate-copy-psp] Error 134

picking fate-h264-timecode, the failure looks like this:

Assertion n <= 31 && value < (1U << n) failed at libavcodec/put_bits.h:169
Aborted (core dumped)
tests/Makefile:250: recipe for target 'fate-h264-timecode' failed
make: *** [fate-h264-timecode] Error 134

#2  0x00005555563156cd in put_bits (s=0x7fffffffd190, n=32, value=990117888) at libavcodec/put_bits.h:169
#3  0x0000555556315ec3 in decode_picture_timing (h=0x5555578239d8, gb=0x7fffffffd200, logctx=0x555557822740) at libavcodec/h264_sei.c:138
#4  0x0000555556316be1 in ff_h264_sei_decode (h=0x5555578239d8, gb=0x7fffffffd2f8, ps=0x555557822f70, logctx=0x555557822740) at libavcodec/h264_sei.c:457
#5  0x0000555556306711 in parse_nal_units (s=0x555557822d80, avctx=0x555557822740, buf=0x555557849090 "", buf_size=7993) at libavcodec/h264_parser.c:333
#6  0x00005555563075ed in h264_parse (s=0x555557822d80, avctx=0x555557822740, poutbuf=0x7fffffffd508, poutbuf_size=0x7fffffffd510, buf=0x555557849090 "", buf_size=7993) at libavcodec/h264_parser.c:616
#7  0x0000555555e40861 in av_parser_parse2 (s=0x555557822d80, avctx=0x555557822740, poutbuf=0x7fffffffd508, poutbuf_size=0x7fffffffd510, buf=0x555557848c40 "f\316\177\307\355'\021\252DOJ)V\207F", buf_size=1024, pts=-9223372036854775808, dts=-9223372036854775808, pos=7168) at libavcodec/parser.c:166
#8  0x0000555555af0a83 in parse_packet (s=0x555557820940, pkt=0x7fffffffd7d0, stream_index=0, flush=0) at libavformat/utils.c:1468
#9  0x0000555555af1828 in read_frame_internal (s=0x555557820940, pkt=0x7fffffffd7d0) at libavformat/utils.c:1682
#10 0x0000555555af9131 in avformat_find_stream_info (ic=0x555557820940, options=0x555557822c80) at libavformat/utils.c:3826
#11 0x0000555555670961 in open_input_file (o=0x7fffffffdb30, filename=0x7fffffffe536 "/home/michael/fatesamples/fate/fate-suite//h264/crew_cif_timecode-2.h264") at fftools/ffmpeg_opt.c:1195
#12 0x000055555567ed59 in open_files (l=0x555557820218, inout=0x5555566262a7 "input", open_file=0x55555566fe3c <open_input_file>) at fftools/ffmpeg_opt.c:3350
#13 0x000055555567ef0d in ffmpeg_parse_options (argc=17, argv=0x7fffffffe198) at fftools/ffmpeg_opt.c:3390
#14 0x000055555569e839 in main (argc=17, argv=0x7fffffffe198) at fftools/ffmpeg.c:4871


[...]
James Almer March 21, 2020, 10:48 p.m. UTC | #9
On 3/21/2020 7:44 PM, Michael Niedermayer wrote:
> On Tue, Mar 17, 2020 at 01:13:53AM +0100, Michael Niedermayer wrote:
>> On Mon, Mar 16, 2020 at 04:42:31PM -0300, James Almer wrote:
>>> On 3/16/2020 3:51 PM, Michael Niedermayer wrote:
>>>> On Fri, Mar 13, 2020 at 11:28:45AM +0100, Anton Khirnov wrote:
>>>>> Those SEIs refer to the currently active SPS. However, since the SEI
>>>>> NALUs precede the coded picture data in the bitstream, the active SPS is
>>>>> in general not known when we are decoding the SEI.
>>>>>
>>>>> Therefore, store the content of the picture timing SEIs and actually
>>>>> parse it when the active SPS is known.
>>>>> ---
>>>>>  libavcodec/h264_parser.c |  9 +++++
>>>>>  libavcodec/h264_sei.c    | 86 ++++++++++++++++++++++++----------------
>>>>>  libavcodec/h264_sei.h    | 11 +++++
>>>>>  libavcodec/h264_slice.c  | 10 +++++
>>>>>  4 files changed, 82 insertions(+), 34 deletions(-)
>>>>
>>>> This seems to break fate
>>>>
>>>> TEST    cbs-h264-CVSE2_Sony_B
>>>> --- ./tests/ref/fate/cbs-h264-CVSE2_Sony_B	2020-03-15 16:08:13.692521532 +0100
>>>> +++ tests/data/fate/cbs-h264-CVSE2_Sony_B	2020-03-16 19:50:23.164302663 +0100
>>>> @@ -1 +0,0 @@
>>>> -ca8bdba497bd2f3b97c50d59692eb537
>>>> Test cbs-h264-CVSE2_Sony_B failed. Look at tests/data/fate/cbs-h264-CVSE2_Sony_B.err for details.
>>>> tests/Makefile:250: recipe for target 'fate-cbs-h264-CVSE2_Sony_B' failed
>>>> make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1
>>>>
>>>> thx
>>>
>>> I can't reproduce this. Are you using a clean tree?
>>
>> i had other patches in there, ill retest without tomorrow (too late now)
>> disregard my comment about there being an issue until i retest
> 
> sorry for te delay
> the failures persist with patches 1 to this
> 
> make: *** [fate-cbs-h264-CVSE2_Sony_B] Error 1
> make: *** [fate-cbs-h264-sei-1] Error 1
> make: *** [fate-h264-timecode] Error 134
> make: *** [fate-h264-mixed-nal-coding] Error 134
> make: *** [fate-h264-twofields-packet] Error 134
> make: *** [fate-h264-attachment-631] Error 134
> make: *** [fate-mov-tenc-only-encrypted] Error 134
> make: *** [fate-mov-frag-encrypted] Error 134
> make: *** [fate-mov-bbi-elst-starts-b] Error 134
> make: *** [fate-h264_mp4toannexb_ticket2991] Error 134
> make: *** [fate-copy-psp] Error 134
> 
> picking fate-h264-timecode, the failure looks like this:
> 
> Assertion n <= 31 && value < (1U << n) failed at libavcodec/put_bits.h:169

This is a level 2 assert. Please make sure to mention when you use
custom configure time options. No wonder we couldn't reproduce it.

> Aborted (core dumped)
> tests/Makefile:250: recipe for target 'fate-h264-timecode' failed
> make: *** [fate-h264-timecode] Error 134
> 
> #2  0x00005555563156cd in put_bits (s=0x7fffffffd190, n=32, value=990117888) at libavcodec/put_bits.h:169
> #3  0x0000555556315ec3 in decode_picture_timing (h=0x5555578239d8, gb=0x7fffffffd200, logctx=0x555557822740) at libavcodec/h264_sei.c:138
> #4  0x0000555556316be1 in ff_h264_sei_decode (h=0x5555578239d8, gb=0x7fffffffd2f8, ps=0x555557822f70, logctx=0x555557822740) at libavcodec/h264_sei.c:457
> #5  0x0000555556306711 in parse_nal_units (s=0x555557822d80, avctx=0x555557822740, buf=0x555557849090 "", buf_size=7993) at libavcodec/h264_parser.c:333
> #6  0x00005555563075ed in h264_parse (s=0x555557822d80, avctx=0x555557822740, poutbuf=0x7fffffffd508, poutbuf_size=0x7fffffffd510, buf=0x555557849090 "", buf_size=7993) at libavcodec/h264_parser.c:616
> #7  0x0000555555e40861 in av_parser_parse2 (s=0x555557822d80, avctx=0x555557822740, poutbuf=0x7fffffffd508, poutbuf_size=0x7fffffffd510, buf=0x555557848c40 "f\316\177\307\355'\021\252DOJ)V\207F", buf_size=1024, pts=-9223372036854775808, dts=-9223372036854775808, pos=7168) at libavcodec/parser.c:166
> #8  0x0000555555af0a83 in parse_packet (s=0x555557820940, pkt=0x7fffffffd7d0, stream_index=0, flush=0) at libavformat/utils.c:1468
> #9  0x0000555555af1828 in read_frame_internal (s=0x555557820940, pkt=0x7fffffffd7d0) at libavformat/utils.c:1682
> #10 0x0000555555af9131 in avformat_find_stream_info (ic=0x555557820940, options=0x555557822c80) at libavformat/utils.c:3826
> #11 0x0000555555670961 in open_input_file (o=0x7fffffffdb30, filename=0x7fffffffe536 "/home/michael/fatesamples/fate/fate-suite//h264/crew_cif_timecode-2.h264") at fftools/ffmpeg_opt.c:1195
> #12 0x000055555567ed59 in open_files (l=0x555557820218, inout=0x5555566262a7 "input", open_file=0x55555566fe3c <open_input_file>) at fftools/ffmpeg_opt.c:3350
> #13 0x000055555567ef0d in ffmpeg_parse_options (argc=17, argv=0x7fffffffe198) at fftools/ffmpeg_opt.c:3390
> #14 0x000055555569e839 in main (argc=17, argv=0x7fffffffe198) at fftools/ffmpeg.c:4871
> 
> 
> [...]
> 
> 
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
Kieran Kunhya March 22, 2020, 4:24 a.m. UTC | #10
On Fri, 13 Mar 2020 at 10:32, Anton Khirnov <anton@khirnov.net> wrote:

> Those SEIs refer to the currently active SPS. However, since the SEI
> NALUs precede the coded picture data in the bitstream, the active SPS is
> in general not known when we are decoding the SEI.
>

The spec disagrees with this statement (7.4.1.2.1 Order of sequence and
picture parameter set RBSPs and their activation). There are reasonably
clear rules about SPS activation.
(Whether real world bitstreams comply is a different issue of course).

Kieran
Anton Khirnov March 23, 2020, 4:17 p.m. UTC | #11
Quoting Kieran Kunhya (2020-03-22 05:24:35)
> On Fri, 13 Mar 2020 at 10:32, Anton Khirnov <anton@khirnov.net> wrote:
> 
> > Those SEIs refer to the currently active SPS. However, since the SEI
> > NALUs precede the coded picture data in the bitstream, the active SPS is
> > in general not known when we are decoding the SEI.
> >
> 
> The spec disagrees with this statement (7.4.1.2.1 Order of sequence and
> picture parameter set RBSPs and their activation). There are reasonably
> clear rules about SPS activation.
> (Whether real world bitstreams comply is a different issue of course).

I don't think you read that section carefully enough. What it says (in
my interpretation) is that an SPS can be activated only by:
- buffering period SEI
- a PPS (which is itself activated by picture data)
Picture timing SEI is neither of those.

Also see D.2.2 Note 1, which suggests exactly what I'm doing with this
patch.
James Almer March 23, 2020, 4:20 p.m. UTC | #12
On 3/23/2020 1:17 PM, Anton Khirnov wrote:
> Quoting Kieran Kunhya (2020-03-22 05:24:35)
>> On Fri, 13 Mar 2020 at 10:32, Anton Khirnov <anton@khirnov.net> wrote:
>>
>>> Those SEIs refer to the currently active SPS. However, since the SEI
>>> NALUs precede the coded picture data in the bitstream, the active SPS is
>>> in general not known when we are decoding the SEI.
>>>
>>
>> The spec disagrees with this statement (7.4.1.2.1 Order of sequence and
>> picture parameter set RBSPs and their activation). There are reasonably
>> clear rules about SPS activation.
>> (Whether real world bitstreams comply is a different issue of course).
> 
> I don't think you read that section carefully enough. What it says (in
> my interpretation) is that an SPS can be activated only by:
> - buffering period SEI

Looking at our Buffering Period SEI parsing code, it doesn't look like
we're activating the SPS it references, for that matter.

> - a PPS (which is itself activated by picture data)
> Picture timing SEI is neither of those.
> 
> Also see D.2.2 Note 1, which suggests exactly what I'm doing with this
> patch.
>
Anton Khirnov March 23, 2020, 4:42 p.m. UTC | #13
Quoting James Almer (2020-03-23 17:20:56)
> On 3/23/2020 1:17 PM, Anton Khirnov wrote:
> > Quoting Kieran Kunhya (2020-03-22 05:24:35)
> >> On Fri, 13 Mar 2020 at 10:32, Anton Khirnov <anton@khirnov.net> wrote:
> >>
> >>> Those SEIs refer to the currently active SPS. However, since the SEI
> >>> NALUs precede the coded picture data in the bitstream, the active SPS is
> >>> in general not known when we are decoding the SEI.
> >>>
> >>
> >> The spec disagrees with this statement (7.4.1.2.1 Order of sequence and
> >> picture parameter set RBSPs and their activation). There are reasonably
> >> clear rules about SPS activation.
> >> (Whether real world bitstreams comply is a different issue of course).
> > 
> > I don't think you read that section carefully enough. What it says (in
> > my interpretation) is that an SPS can be activated only by:
> > - buffering period SEI
> 
> Looking at our Buffering Period SEI parsing code, it doesn't look like
> we're activating the SPS it references, for that matter.

Right, but then against we're not really using it for anything so it
probably doesn't matter.
Kieran Kunhya March 23, 2020, 9:34 p.m. UTC | #14
>
> I don't think you read that section carefully enough. What it says (in
> my interpretation) is that an SPS can be activated only by:
> - buffering period SEI
> - a PPS (which is itself activated by picture data)
> Picture timing SEI is neither of those.
>

The contents of pic-struct do not affect activation but have dependent
syntax elements.
So for example a Buffering Period SEI could activate an SPS and then
followed by pic-struct and pic struct would then have a dependency on the
activated SPS.
This SPS could be different to the one the VCL data refers to. I think
there are other corner cases as well.

Kieran
Anton Khirnov March 24, 2020, 9:39 a.m. UTC | #15
Quoting Kieran Kunhya (2020-03-23 22:34:59)
> >
> > I don't think you read that section carefully enough. What it says (in
> > my interpretation) is that an SPS can be activated only by:
> > - buffering period SEI
> > - a PPS (which is itself activated by picture data)
> > Picture timing SEI is neither of those.
> >
> 
> The contents of pic-struct do not affect activation but have dependent
> syntax elements.
> So for example a Buffering Period SEI could activate an SPS and then
> followed by pic-struct and pic struct would then have a dependency on the
> activated SPS.
> This SPS could be different to the one the VCL data refers to. I think
> there are other corner cases as well.

I don't think it could be different. The active SPS must not change in a
sequence and the SEI messages are all a part of the same sequence as the
coded pictures the belong to. So even if there was an buffering period
SEI that activates some SPS, it has to be the same SPS that will be used
for decoding the slices.
diff mbox series

Patch

diff --git a/libavcodec/h264_parser.c b/libavcodec/h264_parser.c
index 5f9a9c46ef..ec1cbc6a66 100644
--- a/libavcodec/h264_parser.c
+++ b/libavcodec/h264_parser.c
@@ -481,6 +481,15 @@  static inline int parse_nal_units(AVCodecParserContext *s,
                 }
             }
 
+            if (p->sei.picture_timing.present) {
+                ret = ff_h264_sei_process_picture_timing(&p->sei.picture_timing,
+                                                         sps, avctx);
+                if (ret < 0) {
+                    av_log(avctx, AV_LOG_ERROR, "Error processing the picture timing SEI\n");
+                    p->sei.picture_timing.present = 0;
+                }
+            }
+
             if (sps->pic_struct_present_flag && p->sei.picture_timing.present) {
                 switch (p->sei.picture_timing.pic_struct) {
                 case H264_SEI_PIC_STRUCT_TOP_FIELD:
diff --git a/libavcodec/h264_sei.c b/libavcodec/h264_sei.c
index a2452141ca..4f536f82f2 100644
--- a/libavcodec/h264_sei.c
+++ b/libavcodec/h264_sei.c
@@ -54,30 +54,22 @@  void ff_h264_sei_uninit(H264SEIContext *h)
     av_buffer_unref(&h->a53_caption.buf_ref);
 }
 
-static int decode_picture_timing(H264SEIPictureTiming *h, GetBitContext *gb,
-                                 const H264ParamSets *ps, void *logctx)
+int ff_h264_sei_process_picture_timing(H264SEIPictureTiming *h, const SPS *sps,
+                                       void *logctx)
 {
-    int i;
-    const SPS *sps = ps->sps;
-
-    for (i = 0; i<MAX_SPS_COUNT; i++)
-        if ((!sps || !sps->log2_max_frame_num) && ps->sps_list[i])
-            sps = (const SPS *)ps->sps_list[i]->data;
+    GetBitContext gb;
 
-    if (!sps) {
-        av_log(logctx, AV_LOG_ERROR, "SPS unavailable in decode_picture_timing\n");
-        return AVERROR_PS_NOT_FOUND;
-    }
+    init_get_bits(&gb, h->payload, h->payload_size_bits);
 
     if (sps->nal_hrd_parameters_present_flag ||
         sps->vcl_hrd_parameters_present_flag) {
-        h->cpb_removal_delay = get_bits_long(gb, sps->cpb_removal_delay_length);
-        h->dpb_output_delay  = get_bits_long(gb, sps->dpb_output_delay_length);
+        h->cpb_removal_delay = get_bits_long(&gb, sps->cpb_removal_delay_length);
+        h->dpb_output_delay  = get_bits_long(&gb, sps->dpb_output_delay_length);
     }
     if (sps->pic_struct_present_flag) {
         unsigned int i, num_clock_ts;
 
-        h->pic_struct = get_bits(gb, 4);
+        h->pic_struct = get_bits(&gb, 4);
         h->ct_type    = 0;
 
         if (h->pic_struct > H264_SEI_PIC_STRUCT_FRAME_TRIPLING)
@@ -86,38 +78,38 @@  static int decode_picture_timing(H264SEIPictureTiming *h, GetBitContext *gb,
         num_clock_ts = sei_num_clock_ts_table[h->pic_struct];
         h->timecode_cnt = 0;
         for (i = 0; i < num_clock_ts; i++) {
-            if (get_bits(gb, 1)) {                      /* clock_timestamp_flag */
+            if (get_bits(&gb, 1)) {                      /* clock_timestamp_flag */
                 H264SEITimeCode *tc = &h->timecode[h->timecode_cnt++];
                 unsigned int full_timestamp_flag;
                 unsigned int counting_type, cnt_dropped_flag;
-                h->ct_type |= 1 << get_bits(gb, 2);
-                skip_bits(gb, 1);                       /* nuit_field_based_flag */
-                counting_type = get_bits(gb, 5);        /* counting_type */
-                full_timestamp_flag = get_bits(gb, 1);
-                skip_bits(gb, 1);                       /* discontinuity_flag */
-                cnt_dropped_flag = get_bits(gb, 1);      /* cnt_dropped_flag */
+                h->ct_type |= 1 << get_bits(&gb, 2);
+                skip_bits(&gb, 1);                       /* nuit_field_based_flag */
+                counting_type = get_bits(&gb, 5);        /* counting_type */
+                full_timestamp_flag = get_bits(&gb, 1);
+                skip_bits(&gb, 1);                       /* discontinuity_flag */
+                cnt_dropped_flag = get_bits(&gb, 1);      /* cnt_dropped_flag */
                 if (cnt_dropped_flag && counting_type > 1 && counting_type < 7)
                     tc->dropframe = 1;
-                tc->frame = get_bits(gb, 8);         /* n_frames */
+                tc->frame = get_bits(&gb, 8);         /* n_frames */
                 if (full_timestamp_flag) {
                     tc->full = 1;
-                    tc->seconds = get_bits(gb, 6); /* seconds_value 0..59 */
-                    tc->minutes = get_bits(gb, 6); /* minutes_value 0..59 */
-                    tc->hours = get_bits(gb, 5);   /* hours_value 0..23 */
+                    tc->seconds = get_bits(&gb, 6); /* seconds_value 0..59 */
+                    tc->minutes = get_bits(&gb, 6); /* minutes_value 0..59 */
+                    tc->hours = get_bits(&gb, 5);   /* hours_value 0..23 */
                 } else {
                     tc->seconds = tc->minutes = tc->hours = tc->full = 0;
-                    if (get_bits(gb, 1)) {             /* seconds_flag */
-                        tc->seconds = get_bits(gb, 6);
-                        if (get_bits(gb, 1)) {         /* minutes_flag */
-                            tc->minutes = get_bits(gb, 6);
-                            if (get_bits(gb, 1))       /* hours_flag */
-                                tc->hours = get_bits(gb, 5);
+                    if (get_bits(&gb, 1)) {             /* seconds_flag */
+                        tc->seconds = get_bits(&gb, 6);
+                        if (get_bits(&gb, 1)) {         /* minutes_flag */
+                            tc->minutes = get_bits(&gb, 6);
+                            if (get_bits(&gb, 1))       /* hours_flag */
+                                tc->hours = get_bits(&gb, 5);
                         }
                     }
                 }
 
                 if (sps->time_offset_length > 0)
-                    skip_bits(gb,
+                    skip_bits(&gb,
                               sps->time_offset_length); /* time_offset */
             }
         }
@@ -126,6 +118,32 @@  static int decode_picture_timing(H264SEIPictureTiming *h, GetBitContext *gb,
                h->ct_type, h->pic_struct);
     }
 
+    return 0;
+}
+
+static int decode_picture_timing(H264SEIPictureTiming *h, GetBitContext *gb,
+                                 void *logctx)
+{
+    PutBitContext pb;
+    int n;
+
+    if (get_bits_left(gb) > sizeof(h->payload) * 8) {
+        av_log(logctx, AV_LOG_ERROR, "Picture timing SEI payload too large\n");
+        return AVERROR_INVALIDDATA;
+    }
+
+    init_put_bits(&pb, h->payload, sizeof(h->payload));
+
+    while (get_bits_left(gb) >= 32)
+        put_bits(&pb, 32, get_bits_long(gb, 32));
+
+    n = get_bits_left(gb);
+    if (n)
+        put_bits(&pb, n, get_bits_long(gb, n));
+
+    h->payload_size_bits = put_bits_count(&pb);
+    flush_put_bits(&pb);
+
     h->present = 1;
     return 0;
 }
@@ -436,7 +454,7 @@  int ff_h264_sei_decode(H264SEIContext *h, GetBitContext *gb,
 
         switch (type) {
         case H264_SEI_TYPE_PIC_TIMING: // Picture timing SEI
-            ret = decode_picture_timing(&h->picture_timing, &gb_payload, ps, logctx);
+            ret = decode_picture_timing(&h->picture_timing, &gb_payload, logctx);
             break;
         case H264_SEI_TYPE_USER_DATA_REGISTERED:
             ret = decode_registered_user_data(h, &gb_payload, logctx, size);
diff --git a/libavcodec/h264_sei.h b/libavcodec/h264_sei.h
index a75c3aa175..f07a5055c3 100644
--- a/libavcodec/h264_sei.h
+++ b/libavcodec/h264_sei.h
@@ -20,6 +20,7 @@ 
 #define AVCODEC_H264_SEI_H
 
 #include "get_bits.h"
+#include "h264_ps.h"
 
 /**
  * SEI message types
@@ -79,6 +80,10 @@  typedef struct H264SEITimeCode {
 } H264SEITimeCode;
 
 typedef struct H264SEIPictureTiming {
+    // maximum size of pic_timing according to the spec should be 274 bits
+    uint8_t payload[40];
+    int     payload_size_bits;
+
     int present;
     H264_SEI_PicStructType pic_struct;
 
@@ -202,4 +207,10 @@  void ff_h264_sei_uninit(H264SEIContext *h);
  */
 const char *ff_h264_sei_stereo_mode(const H264SEIFramePacking *h);
 
+/**
+ * Parse the contents of a picture timing message given an active SPS.
+ */
+int ff_h264_sei_process_picture_timing(H264SEIPictureTiming *h, const SPS *sps,
+                                       void *logctx);
+
 #endif /* AVCODEC_H264_SEI_H */
diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
index ed4f711ac5..120f282c0a 100644
--- a/libavcodec/h264_slice.c
+++ b/libavcodec/h264_slice.c
@@ -1134,6 +1134,16 @@  static int h264_export_frame_props(H264Context *h)
     /* Signal interlacing information externally. */
     /* Prioritize picture timing SEI information over used
      * decoding process if it exists. */
+    if (h->sei.picture_timing.present) {
+        int ret = ff_h264_sei_process_picture_timing(&h->sei.picture_timing, sps,
+                                                     h->avctx);
+        if (ret < 0) {
+            av_log(h->avctx, AV_LOG_ERROR, "Error processing a picture timing SEI\n");
+            if (h->avctx->err_recognition & AV_EF_EXPLODE)
+                return ret;
+            h->sei.picture_timing.present = 0;
+        }
+    }
 
     if (sps->pic_struct_present_flag && h->sei.picture_timing.present) {
         H264SEIPictureTiming *pt = &h->sei.picture_timing;