| Message ID | 20190514215422.11996-1-andriy.gelman@gmail.com |
|---|---|
| State | New |
| Headers | show |
On Tue, May 14, 2019 at 05:54:21PM -0400, Andriy Gelman wrote: > From: Andriy Gelman <andriy.gelman@gmail.com> > > Fixes Ticket #7895. > > Currently, timestamp interpolation is disabled by default in H264 and > HEVC. This creates playback issues when the demuxer does not output a > valid timestamp. This patch allows interpolation when no b-frames have > been observed during decoding, which fixes playback issues for some > missing timestamp cases. > --- > libavformat/utils.c | 11 +++++++++-- > 1 file changed, 9 insertions(+), 2 deletions(-) > > diff --git a/libavformat/utils.c b/libavformat/utils.c > index a63d71b0f4..3dd0bf0d66 100644 > --- a/libavformat/utils.c > +++ b/libavformat/utils.c > @@ -1233,7 +1233,9 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > int64_t offset; > AVRational duration; > int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 && > - st->codecpar->codec_id != AV_CODEC_ID_HEVC; > + st->codecpar->codec_id != AV_CODEC_ID_HEVC || > + (!st->internal->avctx->max_b_frames && > + st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */ > > if (s->flags & AVFMT_FLAG_NOFILLIN) > return; > @@ -1272,6 +1274,10 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > delay = st->internal->avctx->has_b_frames; > presentation_delayed = 0; > > + /*update max_b_frames if delay is larger */ > + if (delay > st->internal->avctx->max_b_frames) > + st->internal->avctx->max_b_frames = delay; > + > /* XXX: need has_b_frame, but cannot get it if the codec is > * not initialized */ > if (delay && > @@ -1337,7 +1343,8 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts), > pkt->stream_index, pc, pkt->duration, delay, onein_oneout); > > - /* Interpolate PTS and DTS if they are not present. We skip H264 > + /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are > + * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC > * currently because delay and has_b_frames are not reliably set. */ > if ((delay == 0 || (delay == 1 && pc)) && > onein_oneout) { This may produce some approximate values that are sometimes wrong. If you want to compute exact timestamps, see "Conditional coding of timestamps" in H.222. I only have a older draft here so mine only covers mpeg & h264 but maybe the next covers hevc too. [...]
On Thu, 16. May 00:43, Michael Niedermayer wrote: > On Tue, May 14, 2019 at 05:54:21PM -0400, Andriy Gelman wrote: > > From: Andriy Gelman <andriy.gelman@gmail.com> > > > > Fixes Ticket #7895. > > > > Currently, timestamp interpolation is disabled by default in H264 and > > HEVC. This creates playback issues when the demuxer does not output a > > valid timestamp. This patch allows interpolation when no b-frames have > > been observed during decoding, which fixes playback issues for some > > missing timestamp cases. > > --- > > libavformat/utils.c | 11 +++++++++-- > > 1 file changed, 9 insertions(+), 2 deletions(-) > > > > diff --git a/libavformat/utils.c b/libavformat/utils.c > > index a63d71b0f4..3dd0bf0d66 100644 > > --- a/libavformat/utils.c > > +++ b/libavformat/utils.c > > @@ -1233,7 +1233,9 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > int64_t offset; > > AVRational duration; > > int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 && > > - st->codecpar->codec_id != AV_CODEC_ID_HEVC; > > + st->codecpar->codec_id != AV_CODEC_ID_HEVC || > > + (!st->internal->avctx->max_b_frames && > > + st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */ > > > > if (s->flags & AVFMT_FLAG_NOFILLIN) > > return; > > @@ -1272,6 +1274,10 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > delay = st->internal->avctx->has_b_frames; > > presentation_delayed = 0; > > > > + /*update max_b_frames if delay is larger */ > > + if (delay > st->internal->avctx->max_b_frames) > > + st->internal->avctx->max_b_frames = delay; > > + > > /* XXX: need has_b_frame, but cannot get it if the codec is > > * not initialized */ > > if (delay && > > @@ -1337,7 +1343,8 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts), > > pkt->stream_index, pc, pkt->duration, delay, onein_oneout); > > > > - /* Interpolate PTS and DTS if they are not present. We skip H264 > > + /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are > > + * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC > > * currently because delay and has_b_frames are not reliably set. */ > > if ((delay == 0 || (delay == 1 && pc)) && > > onein_oneout) { > > This may produce some approximate values that are sometimes wrong. > > If you want to compute exact timestamps, see "Conditional coding of timestamps" > in H.222. I only have a older draft here so mine only covers mpeg & h264 but > maybe the next covers hevc too. ok, thanks. I'll look over this. Andriy
Michael, On Thu, 16. May 00:43, Michael Niedermayer wrote: > On Tue, May 14, 2019 at 05:54:21PM -0400, Andriy Gelman wrote: > > From: Andriy Gelman <andriy.gelman@gmail.com> > > > > Fixes Ticket #7895. > > > > Currently, timestamp interpolation is disabled by default in H264 and > > HEVC. This creates playback issues when the demuxer does not output a > > valid timestamp. This patch allows interpolation when no b-frames have > > been observed during decoding, which fixes playback issues for some > > missing timestamp cases. > > --- > > libavformat/utils.c | 11 +++++++++-- > > 1 file changed, 9 insertions(+), 2 deletions(-) > > > > diff --git a/libavformat/utils.c b/libavformat/utils.c > > index a63d71b0f4..3dd0bf0d66 100644 > > --- a/libavformat/utils.c > > +++ b/libavformat/utils.c > > @@ -1233,7 +1233,9 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > int64_t offset; > > AVRational duration; > > int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 && > > - st->codecpar->codec_id != AV_CODEC_ID_HEVC; > > + st->codecpar->codec_id != AV_CODEC_ID_HEVC || > > + (!st->internal->avctx->max_b_frames && > > + st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */ > > > > if (s->flags & AVFMT_FLAG_NOFILLIN) > > return; > > @@ -1272,6 +1274,10 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > delay = st->internal->avctx->has_b_frames; > > presentation_delayed = 0; > > > > + /*update max_b_frames if delay is larger */ > > + if (delay > st->internal->avctx->max_b_frames) > > + st->internal->avctx->max_b_frames = delay; > > + > > /* XXX: need has_b_frame, but cannot get it if the codec is > > * not initialized */ > > if (delay && > > @@ -1337,7 +1343,8 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts), > > pkt->stream_index, pc, pkt->duration, delay, onein_oneout); > > > > - /* Interpolate PTS and DTS if they are not present. We skip H264 > > + /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are > > + * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC > > * currently because delay and has_b_frames are not reliably set. */ > > if ((delay == 0 || (delay == 1 && pc)) && > > onein_oneout) { > > This may produce some approximate values that are sometimes wrong. > > If you want to compute exact timestamps, see "Conditional coding of timestamps" > in H.222. I only have a older draft here so mine only covers mpeg & h264 but > maybe the next covers hevc too. In H.222 it says: "The decoding time tdn(j) is specified in the DTS or PTS fields (refer to 2.4.3.6). Decoding times tdn(j + 1), tdn(j + 2), ... of access units without encoded DTS or PTS fields which directly follow access unit j may be derived from information in the elementary stream" If we decode the POC, then we compute the correct pts/dts even for B-frames. Would you agree with this strategy, or did you have something else in mind. Thanks, Andriy
On Sun, Jun 16, 2019 at 01:09:45AM -0400, Andriy Gelman wrote: > Michael, > > On Thu, 16. May 00:43, Michael Niedermayer wrote: > > On Tue, May 14, 2019 at 05:54:21PM -0400, Andriy Gelman wrote: > > > From: Andriy Gelman <andriy.gelman@gmail.com> > > > > > > Fixes Ticket #7895. > > > > > > Currently, timestamp interpolation is disabled by default in H264 and > > > HEVC. This creates playback issues when the demuxer does not output a > > > valid timestamp. This patch allows interpolation when no b-frames have > > > been observed during decoding, which fixes playback issues for some > > > missing timestamp cases. > > > --- > > > libavformat/utils.c | 11 +++++++++-- > > > 1 file changed, 9 insertions(+), 2 deletions(-) > > > > > > diff --git a/libavformat/utils.c b/libavformat/utils.c > > > index a63d71b0f4..3dd0bf0d66 100644 > > > --- a/libavformat/utils.c > > > +++ b/libavformat/utils.c > > > @@ -1233,7 +1233,9 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > > int64_t offset; > > > AVRational duration; > > > int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 && > > > - st->codecpar->codec_id != AV_CODEC_ID_HEVC; > > > + st->codecpar->codec_id != AV_CODEC_ID_HEVC || > > > + (!st->internal->avctx->max_b_frames && > > > + st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */ > > > > > > if (s->flags & AVFMT_FLAG_NOFILLIN) > > > return; > > > @@ -1272,6 +1274,10 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > > delay = st->internal->avctx->has_b_frames; > > > presentation_delayed = 0; > > > > > > + /*update max_b_frames if delay is larger */ > > > + if (delay > st->internal->avctx->max_b_frames) > > > + st->internal->avctx->max_b_frames = delay; > > > + > > > /* XXX: need has_b_frame, but cannot get it if the codec is > > > * not initialized */ > > > if (delay && > > > @@ -1337,7 +1343,8 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, > > > presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts), > > > pkt->stream_index, pc, pkt->duration, delay, onein_oneout); > > > > > > - /* Interpolate PTS and DTS if they are not present. We skip H264 > > > + /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are > > > + * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC > > > * currently because delay and has_b_frames are not reliably set. */ > > > if ((delay == 0 || (delay == 1 && pc)) && > > > onein_oneout) { > > > > This may produce some approximate values that are sometimes wrong. > > > > If you want to compute exact timestamps, see "Conditional coding of timestamps" > > in H.222. I only have a older draft here so mine only covers mpeg & h264 but > > maybe the next covers hevc too. > > In H.222 it says: > > "The decoding time tdn(j) is specified in the DTS or PTS fields (refer to > 2.4.3.6). Decoding times tdn(j + 1), tdn(j + 2), ... of access units without > encoded DTS or PTS fields which directly follow access unit j may be derived > from information in the elementary stream" > > If we decode the POC, then we compute the correct pts/dts even for B-frames. > Would you agree with this strategy, or did you have something else in mind. The POC would provide the ordering but not the exact timestamps Also see "2.7.5 Conditional coding of timestamps" Its a bit ugly sadly, there are many cases, or basically probably every way by which codec level timing can be provided [...]
diff --git a/libavformat/utils.c b/libavformat/utils.c index a63d71b0f4..3dd0bf0d66 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -1233,7 +1233,9 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, int64_t offset; AVRational duration; int onein_oneout = st->codecpar->codec_id != AV_CODEC_ID_H264 && - st->codecpar->codec_id != AV_CODEC_ID_HEVC; + st->codecpar->codec_id != AV_CODEC_ID_HEVC || + (!st->internal->avctx->max_b_frames && + st->cur_dts != RELATIVE_TS_BASE); /*skip when no timing info from demuxer */ if (s->flags & AVFMT_FLAG_NOFILLIN) return; @@ -1272,6 +1274,10 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, delay = st->internal->avctx->has_b_frames; presentation_delayed = 0; + /*update max_b_frames if delay is larger */ + if (delay > st->internal->avctx->max_b_frames) + st->internal->avctx->max_b_frames = delay; + /* XXX: need has_b_frame, but cannot get it if the codec is * not initialized */ if (delay && @@ -1337,7 +1343,8 @@ static void compute_pkt_fields(AVFormatContext *s, AVStream *st, presentation_delayed, av_ts2str(pkt->pts), av_ts2str(pkt->dts), av_ts2str(st->cur_dts), pkt->stream_index, pc, pkt->duration, delay, onein_oneout); - /* Interpolate PTS and DTS if they are not present. We skip H264 + /* Interpolate PTS and DTS if they are not present. H264/HEVC timestamps are + * interpolated only if no b-frames have been observed. Otherwise, we skip H264/HEVC * currently because delay and has_b_frames are not reliably set. */ if ((delay == 0 || (delay == 1 && pc)) && onein_oneout) {
From: Andriy Gelman <andriy.gelman@gmail.com> Fixes Ticket #7895. Currently, timestamp interpolation is disabled by default in H264 and HEVC. This creates playback issues when the demuxer does not output a valid timestamp. This patch allows interpolation when no b-frames have been observed during decoding, which fixes playback issues for some missing timestamp cases. --- libavformat/utils.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)