[FFmpeg-devel,v2,1/2] libavformat/utils: Interpolate missing timestamps in H264 and HEVC when no b-frames observed

Submitted by Andriy Gelman on May 14, 2019, 9:54 p.m.

Details

Message ID 20190514215422.11996-1-andriy.gelman@gmail.com
State New
Headers show

Commit Message

Andriy Gelman May 14, 2019, 9:54 p.m.
From: Andriy Gelman <andriy.gelman@gmail.com>

Fixes Ticket #7895.

Currently, timestamp interpolation is disabled by default in H264 and
HEVC.  This creates playback issues when the demuxer does not output a
valid timestamp. This patch allows interpolation when no b-frames have
been observed during decoding, which fixes playback issues for some
missing timestamp cases.
---
 libavformat/utils.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

Comments

Michael Niedermayer May 15, 2019, 10:43 p.m.
On Tue, May 14, 2019 at 05:54:21PM -0400, Andriy Gelman wrote:
> From: Andriy Gelman <andriy.gelman@gmail.com>
> 
> Fixes Ticket #7895.
> 
> Currently, timestamp interpolation is disabled by default in H264 and
> HEVC.  This creates playback issues when the demuxer does not output a
> valid timestamp. This patch allows interpolation when no b-frames have
> been observed during decoding, which fixes playback issues for some
> missing timestamp cases.
> ---
>  libavformat/utils.c | 11 +++++++++--
>  1 file changed, 9 insertions(+), 2 deletions(-)
> 
> diff --git a/libavformat/utils.c b/libavformat/utils.c
> index a63d71b0f4..3dd0bf0d66 100644
> --- a/libavformat/utils.c
> +++ b/libavformat/utils.c
> @@ -1233,7 +1233,9 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
>      int64_t offset;
>      AVRational duration;
>      int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 &&
> -                       st->codecpar->codec_id != AV_CODEC_ID_HEVC;
> +                       st->codecpar->codec_id != AV_CODEC_ID_HEVC ||
> +                       (!st->internal->avctx->max_b_frames &&
> +                        st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */
>  
>      if (s->flags & AVFMT_FLAG_NOFILLIN)
>          return;
> @@ -1272,6 +1274,10 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
>      delay = st->internal->avctx->has_b_frames;
>      presentation_delayed = 0;
>  
> +    /*update max_b_frames if delay is larger */
> +    if (delay > st->internal->avctx->max_b_frames)
> +      st->internal->avctx->max_b_frames = delay;
> +
>      /* XXX: need has_b_frame, but cannot get it if the codec is
>       *  not initialized */
>      if (delay &&
> @@ -1337,7 +1343,8 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
>              presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts),
>              pkt->stream_index, pc, pkt->duration, delay, onein_oneout);
>  
> -    /* Interpolate PTS and DTS if they are not present. We skip H264
> +    /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are
> +     * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC
>       * currently because delay and has_b_frames are not reliably set. */
>      if ((delay == 0 || (delay == 1 && pc)) &&
>          onein_oneout) {

This may produce some approximate values that are sometimes wrong.

If you want to compute exact timestamps, see "Conditional coding of timestamps"
in H.222. I only have a older draft here so mine only covers mpeg & h264 but 
maybe the next covers hevc too.

[...]
Andriy Gelman May 16, 2019, 12:09 a.m.
On Thu, 16. May 00:43, Michael Niedermayer wrote:
> On Tue, May 14, 2019 at 05:54:21PM -0400, Andriy Gelman wrote:
> > From: Andriy Gelman <andriy.gelman@gmail.com>
> > 
> > Fixes Ticket #7895.
> > 
> > Currently, timestamp interpolation is disabled by default in H264 and
> > HEVC.  This creates playback issues when the demuxer does not output a
> > valid timestamp. This patch allows interpolation when no b-frames have
> > been observed during decoding, which fixes playback issues for some
> > missing timestamp cases.
> > ---
> >  libavformat/utils.c | 11 +++++++++--
> >  1 file changed, 9 insertions(+), 2 deletions(-)
> > 
> > diff --git a/libavformat/utils.c b/libavformat/utils.c
> > index a63d71b0f4..3dd0bf0d66 100644
> > --- a/libavformat/utils.c
> > +++ b/libavformat/utils.c
> > @@ -1233,7 +1233,9 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
> >      int64_t offset;
> >      AVRational duration;
> >      int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 &&
> > -                       st->codecpar->codec_id != AV_CODEC_ID_HEVC;
> > +                       st->codecpar->codec_id != AV_CODEC_ID_HEVC ||
> > +                       (!st->internal->avctx->max_b_frames &&
> > +                        st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */
> >  
> >      if (s->flags & AVFMT_FLAG_NOFILLIN)
> >          return;
> > @@ -1272,6 +1274,10 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
> >      delay = st->internal->avctx->has_b_frames;
> >      presentation_delayed = 0;
> >  
> > +    /*update max_b_frames if delay is larger */
> > +    if (delay > st->internal->avctx->max_b_frames)
> > +      st->internal->avctx->max_b_frames = delay;
> > +
> >      /* XXX: need has_b_frame, but cannot get it if the codec is
> >       *  not initialized */
> >      if (delay &&
> > @@ -1337,7 +1343,8 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
> >              presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts),
> >              pkt->stream_index, pc, pkt->duration, delay, onein_oneout);
> >  
> > -    /* Interpolate PTS and DTS if they are not present. We skip H264
> > +    /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are
> > +     * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC
> >       * currently because delay and has_b_frames are not reliably set. */
> >      if ((delay == 0 || (delay == 1 && pc)) &&
> >          onein_oneout) {
> 
> This may produce some approximate values that are sometimes wrong.
> 
> If you want to compute exact timestamps, see "Conditional coding of timestamps"
> in H.222. I only have a older draft here so mine only covers mpeg & h264 but 
> maybe the next covers hevc too.

ok, thanks. I'll look over this.
Andriy

Patch hide | download patch | download mbox

diff --git a/libavformat/utils.c b/libavformat/utils.c
index a63d71b0f4..3dd0bf0d66 100644
--- a/libavformat/utils.c
+++ b/libavformat/utils.c
@@ -1233,7 +1233,9 @@  static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
     int64_t offset;
     AVRational duration;
     int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 &&
-                       st->codecpar->codec_id != AV_CODEC_ID_HEVC;
+                       st->codecpar->codec_id != AV_CODEC_ID_HEVC ||
+                       (!st->internal->avctx->max_b_frames &&
+                        st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */
 
     if (s->flags & AVFMT_FLAG_NOFILLIN)
         return;
@@ -1272,6 +1274,10 @@  static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
     delay = st->internal->avctx->has_b_frames;
     presentation_delayed = 0;
 
+    /*update max_b_frames if delay is larger */
+    if (delay > st->internal->avctx->max_b_frames)
+      st->internal->avctx->max_b_frames = delay;
+
     /* XXX: need has_b_frame, but cannot get it if the codec is
      *  not initialized */
     if (delay &&
@@ -1337,7 +1343,8 @@  static void compute_pkt_fields(AVFormatContext *s, AVStream *st,
             presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts),
             pkt->stream_index, pc, pkt->duration, delay, onein_oneout);
 
-    /* Interpolate PTS and DTS if they are not present. We skip H264
+    /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are
+     * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC
      * currently because delay and has_b_frames are not reliably set. */
     if ((delay == 0 || (delay == 1 && pc)) &&
         onein_oneout) {