[FFmpeg-devel,v2] avcodec/mediacodecdec: work around for decoding h264 with coded fields

Submitted by Aman Gupta on June 12, 2018, 9:37 p.m.

Details

Message ID 20180612213746.17225-1-ffmpeg@tmm1.net
State New
Headers show

Commit Message

Aman Gupta June 12, 2018, 9:37 p.m.
From: Aman Gupta <aman@tmm1.net>

This is a hacky work-around for #7092, where the lavc h264
parser splits coded fields into separate video packets, only one
of which has a PTS set.

The MediaCodec#queueInputBuffer API expects a PTS along with
incoming video packets, and breaks badly when the PTS is missing
or incorrect (previously it would be passed in as AV_NOPTS_VALUE,
but the same breakage happens if you pass in 0 instead).

Since it seems there's no easy fix for #7092, this patch stores
the previous PTS in the decoder context and re-uses it for the
second packet. This emulates the behavior of other Android video
players that don't split the coded fields, and pass them as a single
buffer with the same timestamp.

Signed-off-by: Aman Gupta <aman@tmm1.net>
---
 libavcodec/mediacodecdec_common.c | 9 +++++++++
 libavcodec/mediacodecdec_common.h | 2 ++
 2 files changed, 11 insertions(+)

Comments

Aman Gupta June 15, 2018, 9:22 p.m.
On Tue, Jun 12, 2018 at 2:37 PM Aman Gupta <ffmpeg@tmm1.net> wrote:

> From: Aman Gupta <aman@tmm1.net>
>
> This is a hacky work-around for #7092, where the lavc h264
> parser splits coded fields into separate video packets, only one
> of which has a PTS set.
>
> The MediaCodec#queueInputBuffer API expects a PTS along with
> incoming video packets, and breaks badly when the PTS is missing
> or incorrect (previously it would be passed in as AV_NOPTS_VALUE,
> but the same breakage happens if you pass in 0 instead).
>
> Since it seems there's no easy fix for #7092, this patch stores
> the previous PTS in the decoder context and re-uses it for the
> second packet. This emulates the behavior of other Android video
>

From the MediaCodec documentation (
https://developer.android.com/reference/android/media/MediaCodec.html):

    Upon obtaining an input buffer, fill it with data and submit it to the
codec using queueInputBuffer – or queueSecureInputBuffer if using
decryption.
    **Do not submit multiple input buffers with the same timestamp**
(unless it is codec-specific data marked as such).

So it seems it is not valid to store and re-use the previous PTS when
submitting a new input buffer. Though it works on some devices, others will
print "Repeating PTS" warnings and corrupt the playback stream.

Ideally the mediacodec decoder could simply ask the parser to pass it the
original video packet without splitting it up, though I'm not sure how
feasible that is.

@JEEB also mentioned a possible work-around where intermediary timestamps
could be generated by looking at the poc.frame_num of the field picture
that is split off into its own packet.

Aman


> players that don't split the coded fields, and pass them as a single
> buffer with the same timestamp.
>
> Signed-off-by: Aman Gupta <aman@tmm1.net>
> ---
>  libavcodec/mediacodecdec_common.c | 9 +++++++++
>  libavcodec/mediacodecdec_common.h | 2 ++
>  2 files changed, 11 insertions(+)
>
> diff --git a/libavcodec/mediacodecdec_common.c
> b/libavcodec/mediacodecdec_common.c
> index 40a2ee6778..80cbb7afbd 100644
> --- a/libavcodec/mediacodecdec_common.c
> +++ b/libavcodec/mediacodecdec_common.c
> @@ -448,6 +448,7 @@ static int mediacodec_dec_flush_codec(AVCodecContext
> *avctx, MediaCodecDecContex
>      s->eos = 0;
>      atomic_fetch_add(&s->serial, 1);
>      atomic_init(&s->hw_buffer_count, 0);
> +    s->last_pts = AV_NOPTS_VALUE;
>      s->current_input_buffer = -1;
>
>      status = ff_AMediaCodec_flush(codec);
> @@ -476,6 +477,7 @@ int ff_mediacodec_dec_init(AVCodecContext *avctx,
> MediaCodecDecContext *s,
>      atomic_init(&s->refcount, 1);
>      atomic_init(&s->hw_buffer_count, 0);
>      atomic_init(&s->serial, 1);
> +    s->last_pts = AV_NOPTS_VALUE;
>      s->current_input_buffer = -1;
>
>      pix_fmt = ff_get_format(avctx, pix_fmts);
> @@ -609,6 +611,13 @@ int ff_mediacodec_dec_send(AVCodecContext *avctx,
> MediaCodecDecContext *s,
>          }
>
>          pts = pkt->pts;
> +        if (pts == AV_NOPTS_VALUE && s->last_pts != AV_NOPTS_VALUE) {
> +            pts = s->last_pts;
> +        } else if (pts == AV_NOPTS_VALUE) {
> +            av_log(avctx, AV_LOG_WARNING, "Packet is missing PTS!\n");
> +            pts = 0;
> +        }
> +        s->last_pts = pkt->pts;
>          if (pts != AV_NOPTS_VALUE && avctx->pkt_timebase.num &&
> avctx->pkt_timebase.den) {
>              pts = av_rescale_q(pts, avctx->pkt_timebase, AV_TIME_BASE_Q);
>          }
> diff --git a/libavcodec/mediacodecdec_common.h
> b/libavcodec/mediacodecdec_common.h
> index d280236b8e..9f22006e12 100644
> --- a/libavcodec/mediacodecdec_common.h
> +++ b/libavcodec/mediacodecdec_common.h
> @@ -69,6 +69,8 @@ typedef struct MediaCodecDecContext {
>      bool delay_flush;
>      atomic_int serial;
>
> +    int64_t last_pts;
> +
>  } MediaCodecDecContext;
>
>  int ff_mediacodec_dec_init(AVCodecContext *avctx,
> --
> 2.14.2
>
>

Patch hide | download patch | download mbox

diff --git a/libavcodec/mediacodecdec_common.c b/libavcodec/mediacodecdec_common.c
index 40a2ee6778..80cbb7afbd 100644
--- a/libavcodec/mediacodecdec_common.c
+++ b/libavcodec/mediacodecdec_common.c
@@ -448,6 +448,7 @@  static int mediacodec_dec_flush_codec(AVCodecContext *avctx, MediaCodecDecContex
     s->eos = 0;
     atomic_fetch_add(&s->serial, 1);
     atomic_init(&s->hw_buffer_count, 0);
+    s->last_pts = AV_NOPTS_VALUE;
     s->current_input_buffer = -1;
 
     status = ff_AMediaCodec_flush(codec);
@@ -476,6 +477,7 @@  int ff_mediacodec_dec_init(AVCodecContext *avctx, MediaCodecDecContext *s,
     atomic_init(&s->refcount, 1);
     atomic_init(&s->hw_buffer_count, 0);
     atomic_init(&s->serial, 1);
+    s->last_pts = AV_NOPTS_VALUE;
     s->current_input_buffer = -1;
 
     pix_fmt = ff_get_format(avctx, pix_fmts);
@@ -609,6 +611,13 @@  int ff_mediacodec_dec_send(AVCodecContext *avctx, MediaCodecDecContext *s,
         }
 
         pts = pkt->pts;
+        if (pts == AV_NOPTS_VALUE && s->last_pts != AV_NOPTS_VALUE) {
+            pts = s->last_pts;
+        } else if (pts == AV_NOPTS_VALUE) {
+            av_log(avctx, AV_LOG_WARNING, "Packet is missing PTS!\n");
+            pts = 0;
+        }
+        s->last_pts = pkt->pts;
         if (pts != AV_NOPTS_VALUE && avctx->pkt_timebase.num && avctx->pkt_timebase.den) {
             pts = av_rescale_q(pts, avctx->pkt_timebase, AV_TIME_BASE_Q);
         }
diff --git a/libavcodec/mediacodecdec_common.h b/libavcodec/mediacodecdec_common.h
index d280236b8e..9f22006e12 100644
--- a/libavcodec/mediacodecdec_common.h
+++ b/libavcodec/mediacodecdec_common.h
@@ -69,6 +69,8 @@  typedef struct MediaCodecDecContext {
     bool delay_flush;
     atomic_int serial;
 
+    int64_t last_pts;
+
 } MediaCodecDecContext;
 
 int ff_mediacodec_dec_init(AVCodecContext *avctx,