diff mbox

[FFmpeg-devel] lavf/mov.c: Make audio timestamps strictly monotonically increasing inside an edit list. Fixes gapless decoding.

Message ID 1474680018-24328-1-git-send-email-isasi@google.com
State Superseded
Headers show

Commit Message

Sasi Inguva Sept. 24, 2016, 1:20 a.m. UTC 
Signed-off-by: Sasi Inguva <isasi@google.com>
---
 libavcodec/utils.c                           | 16 +++---
 libavformat/mov.c                            | 81 ++++++++++++++++++++++++----
 tests/fate-run.sh                            |  2 +-
 tests/ref/fate/gaplessenc-itunes-to-ipod-aac | 32 +++++------
 tests/ref/fate/gaplessenc-pcm-to-mov-aac     | 32 +++++------
 tests/ref/fate/gaplessinfo-itunes1           | 32 +++++------
 tests/ref/fate/gaplessinfo-itunes2           | 32 +++++------
 7 files changed, 142 insertions(+), 85 deletions(-)

Comments

Michael Niedermayer Sept. 24, 2016, 1:55 a.m. UTC | #1 
On Fri, Sep 23, 2016 at 06:20:18PM -0700, Sasi Inguva wrote:
> Signed-off-by: Sasi Inguva <isasi@google.com>
> ---
>  libavcodec/utils.c                           | 16 +++---
>  libavformat/mov.c                            | 81 ++++++++++++++++++++++++----

>  tests/fate-run.sh                            |  2 +-

this belongs in a seperate patch

libavcodec and libavformat changes should also be split into 2
patches to keep things cleanly seperated

thx

[...]
Sasi Inguva Sept. 26, 2016, 6:46 p.m. UTC | #2 
Sent three separate patches. PTAL. Thanks.

On Fri, Sep 23, 2016 at 6:55 PM, Michael Niedermayer <michael@niedermayer.cc
> wrote:

> On Fri, Sep 23, 2016 at 06:20:18PM -0700, Sasi Inguva wrote:
> > Signed-off-by: Sasi Inguva <isasi@google.com>
> > ---
> >  libavcodec/utils.c                           | 16 +++---
> >  libavformat/mov.c                            | 81
> ++++++++++++++++++++++++----
>
> >  tests/fate-run.sh                            |  2 +-
>
> this belongs in a seperate patch
>
> libavcodec and libavformat changes should also be split into 2
> patches to keep things cleanly seperated
>
> thx
>
> [...]
> --
> Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
>
> Breaking DRM is a little like attempting to break through a door even
> though the window is wide open and the only thing in the house is a bunch
> of things you dont want and which you would get tomorrow for free anyway
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
>
diff mbox

Patch

diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index b0345b6..6323156 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -2320,7 +2320,6 @@  int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
         uint32_t discard_padding = 0;
         uint8_t skip_reason = 0;
         uint8_t discard_reason = 0;
-        int demuxer_skip_samples = 0;
         // copy to ensure we do not change avpkt
         AVPacket tmp = *avpkt;
         int did_split = av_packet_split_side_data(&tmp);
@@ -2328,7 +2327,6 @@  int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
         if (ret < 0)
             goto fail;
 
-        demuxer_skip_samples = avctx->internal->skip_samples;
         avctx->internal->pkt = &tmp;
         if (HAVE_THREADS && avctx->active_thread_type & FF_THREAD_FRAME)
             ret = ff_thread_decode_frame(avctx, frame, got_frame_ptr, &tmp);
@@ -2353,13 +2351,6 @@  int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
                 frame->sample_rate = avctx->sample_rate;
         }
 
-
-        if (frame->flags & AV_FRAME_FLAG_DISCARD) {
-            // If using discard frame flag, ignore skip_samples set by the decoder.
-            avctx->internal->skip_samples = demuxer_skip_samples;
-            *got_frame_ptr = 0;
-        }
-
         side= av_packet_get_side_data(avctx->internal->pkt, AV_PKT_DATA_SKIP_SAMPLES, &side_size);
         if(side && side_size>=10) {
             avctx->internal->skip_samples = AV_RL32(side);
@@ -2369,6 +2360,13 @@  int attribute_align_arg avcodec_decode_audio4(AVCodecContext *avctx,
             skip_reason = AV_RL8(side + 8);
             discard_reason = AV_RL8(side + 9);
         }
+
+        if ((frame->flags & AV_FRAME_FLAG_DISCARD) && *got_frame_ptr &&
+            !(avctx->flags2 & AV_CODEC_FLAG2_SKIP_MANUAL)) {
+            avctx->internal->skip_samples -= frame->nb_samples;
+            *got_frame_ptr = 0;
+        }
+
         if (avctx->internal->skip_samples > 0 && *got_frame_ptr &&
             !(avctx->flags2 & AV_CODEC_FLAG2_SKIP_MANUAL)) {
             if(frame->nb_samples <= avctx->internal->skip_samples){
diff --git a/libavformat/mov.c b/libavformat/mov.c
index b84d9c0..bb86780 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -2856,6 +2856,21 @@  static int64_t add_index_entry(AVStream *st, int64_t pos, int64_t timestamp,
 }
 
 /**
+ * Rewrite timestamps of index entries in the range [end_index - frame_duration_buffer_size, end_index)
+ * by subtracting end_ts successively by the amounts given in frame_duration_buffer.
+ */
+static void fix_index_entry_timestamps(AVStream* st, int end_index, int64_t end_ts,
+                                       int64_t* frame_duration_buffer,
+                                       int frame_duration_buffer_size) {
+    int i = 0;
+    av_assert0(end_index >= 0 && end_index <= st->nb_index_entries);
+    for (i = 0; i < frame_duration_buffer_size; i++) {
+        end_ts -= frame_duration_buffer[frame_duration_buffer_size - 1 - i];
+        st->index_entries[end_index - 1 - i].timestamp = end_ts;
+    }
+}
+
+/**
  * Append a new ctts entry to ctts_data.
  * Returns the new ctts_count if successful, else returns -1.
  */
@@ -2919,7 +2934,10 @@  static void mov_fix_index(MOVContext *mov, AVStream *st)
     int64_t edit_list_media_time_dts = 0;
     int64_t edit_list_start_encountered = 0;
     int64_t search_timestamp = 0;
-
+    int64_t* frame_duration_buffer = NULL;
+    int num_discarded_begin = 0;
+    int first_non_zero_audio_edit = -1;
+    int packet_skip_samples = 0;
 
     if (!msc->elst_data || msc->elst_count <= 0) {
         return;
@@ -2955,6 +2973,7 @@  static void mov_fix_index(MOVContext *mov, AVStream *st)
         edit_list_index++;
         edit_list_dts_counter = edit_list_dts_entry_end;
         edit_list_dts_entry_end += edit_list_duration;
+        num_discarded_begin = 0;
         if (edit_list_media_time == -1) {
             continue;
         }
@@ -2962,7 +2981,14 @@  static void mov_fix_index(MOVContext *mov, AVStream *st)
         // If we encounter a non-negative edit list reset the skip_samples/start_pad fields and set them
         // according to the edit list below.
         if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
-            st->skip_samples = msc->start_pad = 0;
+            if (first_non_zero_audio_edit < 0) {
+                first_non_zero_audio_edit = 1;
+            } else {
+                first_non_zero_audio_edit = 0;
+            }
+
+            if (first_non_zero_audio_edit > 0)
+                st->skip_samples = msc->start_pad = 0;
         }
 
         //find closest previous key frame
@@ -3041,24 +3067,57 @@  static void mov_fix_index(MOVContext *mov, AVStream *st)
             }
 
             if (curr_cts < edit_list_media_time || curr_cts >= (edit_list_duration + edit_list_media_time)) {
-                if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && curr_cts < edit_list_media_time &&
-                    curr_cts + frame_duration > edit_list_media_time &&
-                    st->skip_samples == 0 && msc->start_pad == 0) {
-                    st->skip_samples = msc->start_pad = edit_list_media_time - curr_cts;
-
-                    // Shift the index entry timestamp by skip_samples to be correct.
-                    edit_list_dts_counter -= st->skip_samples;
+                if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && st->codecpar->codec_id != AV_CODEC_ID_VORBIS &&
+                    curr_cts < edit_list_media_time && curr_cts + frame_duration > edit_list_media_time &&
+                    first_non_zero_audio_edit > 0) {
+                     packet_skip_samples = edit_list_media_time - curr_cts;
+                     st->skip_samples += packet_skip_samples;
+
+                    // Shift the index entry timestamp by packet_skip_samples to be correct.
+                    edit_list_dts_counter -= packet_skip_samples;
                     if (edit_list_start_encountered == 0)  {
-                      edit_list_start_encountered = 1;
+                        edit_list_start_encountered = 1;
+                        // Make timestamps strictly monotonically increasing for audio, by rewriting timestamps for
+                        // discarded packets.
+                        if (frame_duration_buffer) {
+                          fix_index_entry_timestamps(st, st->nb_index_entries, edit_list_dts_counter,
+                                                     frame_duration_buffer, num_discarded_begin);
+                          av_freep(&frame_duration_buffer);
+                        }
                     }
 
-                    av_log(mov->fc, AV_LOG_DEBUG, "skip %d audio samples from curr_cts: %"PRId64"\n", st->skip_samples, curr_cts);
+                    av_log(mov->fc, AV_LOG_DEBUG, "skip %d audio samples from curr_cts: %"PRId64"\n", packet_skip_samples, curr_cts);
                 } else {
                     flags |= AVINDEX_DISCARD_FRAME;
                     av_log(mov->fc, AV_LOG_DEBUG, "drop a frame at curr_cts: %"PRId64" @ %"PRId64"\n", curr_cts, index);
+
+                    if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && edit_list_start_encountered == 0) {
+                        num_discarded_begin++;
+                        frame_duration_buffer = av_realloc(frame_duration_buffer,
+                                                           num_discarded_begin * sizeof(int64_t));
+                        if (!frame_duration_buffer) {
+                            av_log(mov->fc, AV_LOG_ERROR, "Cannot reallocate frame duration buffer\n");
+                            break;
+                        }
+                        frame_duration_buffer[num_discarded_begin - 1] = frame_duration;
+
+                        // Increment skip_samples for the first non-zero audio edit list
+                        if (first_non_zero_audio_edit > 0 && st->codecpar->codec_id != AV_CODEC_ID_VORBIS) {
+                            st->skip_samples += frame_duration;
+                            msc->start_pad = st->skip_samples;
+                        }
+                    }
                 }
             } else if (edit_list_start_encountered == 0) {
                 edit_list_start_encountered = 1;
+                // Make timestamps strictly monotonically increasing for audio, by rewriting timestamps for
+                // discarded packets.
+                if (st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && frame_duration_buffer) {
+                    fix_index_entry_timestamps(st, st->nb_index_entries, edit_list_dts_counter,
+                                               frame_duration_buffer, num_discarded_begin);
+                    av_freep(&frame_duration_buffer);
+                }
+
             }
 
             if (add_index_entry(st, current->pos, edit_list_dts_counter, current->size,
diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index c640cc5..cca8fa0 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -107,7 +107,7 @@  probegaplessinfo(){
     pktfile1="${outdir}/${test}.pkts"
     framefile1="${outdir}/${test}.frames"
     cleanfiles="$cleanfiles $pktfile1 $framefile1"
-    run ffprobe${PROGSUF} -bitexact -select_streams a -of compact -count_packets -show_entries packet=pts,dts,duration:stream=nb_read_packets -v 0 "$filename" "$@" > "$pktfile1"
+    run ffprobe${PROGSUF} -bitexact -select_streams a -of compact -count_packets -show_entries packet=pts,dts,duration,flags:stream=nb_read_packets -v 0 "$filename" "$@" > "$pktfile1"
     head -n 8 "$pktfile1"
     tail -n 9 "$pktfile1"
     run ffprobe${PROGSUF} -bitexact -select_streams a -of compact -count_frames -show_entries frame=pkt_pts,pkt_dts,best_effort_timestamp,pkt_duration,nb_samples:stream=nb_read_frames -v 0 "$filename" "$@" > "$framefile1"
diff --git a/tests/ref/fate/gaplessenc-itunes-to-ipod-aac b/tests/ref/fate/gaplessenc-itunes-to-ipod-aac
index 043c085..ed68487 100644
--- a/tests/ref/fate/gaplessenc-itunes-to-ipod-aac
+++ b/tests/ref/fate/gaplessenc-itunes-to-ipod-aac
@@ -7,22 +7,22 @@  duration_ts=103326
 start_time=0.000000
 duration=2.367000
 [/FORMAT]
-packet|pts=0|dts=0|duration=N/A
-packet|pts=0|dts=0|duration=1024
-packet|pts=1024|dts=1024|duration=1024
-packet|pts=2048|dts=2048|duration=1024
-packet|pts=3072|dts=3072|duration=1024
-packet|pts=4096|dts=4096|duration=1024
-packet|pts=5120|dts=5120|duration=1024
-packet|pts=6144|dts=6144|duration=1024
-packet|pts=95232|dts=95232|duration=1024
-packet|pts=96256|dts=96256|duration=1024
-packet|pts=97280|dts=97280|duration=1024
-packet|pts=98304|dts=98304|duration=1024
-packet|pts=99328|dts=99328|duration=1024
-packet|pts=100352|dts=100352|duration=1024
-packet|pts=101376|dts=101376|duration=1024
-packet|pts=102400|dts=102400|duration=926
+packet|pts=-1024|dts=-1024|duration=1024|flags=KD
+packet|pts=0|dts=0|duration=1024|flags=K_
+packet|pts=1024|dts=1024|duration=1024|flags=K_
+packet|pts=2048|dts=2048|duration=1024|flags=K_
+packet|pts=3072|dts=3072|duration=1024|flags=K_
+packet|pts=4096|dts=4096|duration=1024|flags=K_
+packet|pts=5120|dts=5120|duration=1024|flags=K_
+packet|pts=6144|dts=6144|duration=1024|flags=K_
+packet|pts=95232|dts=95232|duration=1024|flags=K_
+packet|pts=96256|dts=96256|duration=1024|flags=K_
+packet|pts=97280|dts=97280|duration=1024|flags=K_
+packet|pts=98304|dts=98304|duration=1024|flags=K_
+packet|pts=99328|dts=99328|duration=1024|flags=K_
+packet|pts=100352|dts=100352|duration=1024|flags=K_
+packet|pts=101376|dts=101376|duration=1024|flags=K_
+packet|pts=102400|dts=102400|duration=926|flags=K_
 stream|nb_read_packets=102
 frame|pkt_pts=0|pkt_dts=0|best_effort_timestamp=0|pkt_duration=1024|nb_samples=1024
 frame|pkt_pts=1024|pkt_dts=1024|best_effort_timestamp=1024|pkt_duration=1024|nb_samples=1024
diff --git a/tests/ref/fate/gaplessenc-pcm-to-mov-aac b/tests/ref/fate/gaplessenc-pcm-to-mov-aac
index 8b7e3f6..b5f9182 100644
--- a/tests/ref/fate/gaplessenc-pcm-to-mov-aac
+++ b/tests/ref/fate/gaplessenc-pcm-to-mov-aac
@@ -7,22 +7,22 @@  duration_ts=529200
 start_time=0.000000
 duration=12.024000
 [/FORMAT]
-packet|pts=0|dts=0|duration=N/A
-packet|pts=0|dts=0|duration=1024
-packet|pts=1024|dts=1024|duration=1024
-packet|pts=2048|dts=2048|duration=1024
-packet|pts=3072|dts=3072|duration=1024
-packet|pts=4096|dts=4096|duration=1024
-packet|pts=5120|dts=5120|duration=1024
-packet|pts=6144|dts=6144|duration=1024
-packet|pts=521216|dts=521216|duration=1024
-packet|pts=522240|dts=522240|duration=1024
-packet|pts=523264|dts=523264|duration=1024
-packet|pts=524288|dts=524288|duration=1024
-packet|pts=525312|dts=525312|duration=1024
-packet|pts=526336|dts=526336|duration=1024
-packet|pts=527360|dts=527360|duration=1024
-packet|pts=528384|dts=528384|duration=816
+packet|pts=-1024|dts=-1024|duration=1024|flags=KD
+packet|pts=0|dts=0|duration=1024|flags=K_
+packet|pts=1024|dts=1024|duration=1024|flags=K_
+packet|pts=2048|dts=2048|duration=1024|flags=K_
+packet|pts=3072|dts=3072|duration=1024|flags=K_
+packet|pts=4096|dts=4096|duration=1024|flags=K_
+packet|pts=5120|dts=5120|duration=1024|flags=K_
+packet|pts=6144|dts=6144|duration=1024|flags=K_
+packet|pts=521216|dts=521216|duration=1024|flags=K_
+packet|pts=522240|dts=522240|duration=1024|flags=K_
+packet|pts=523264|dts=523264|duration=1024|flags=K_
+packet|pts=524288|dts=524288|duration=1024|flags=K_
+packet|pts=525312|dts=525312|duration=1024|flags=K_
+packet|pts=526336|dts=526336|duration=1024|flags=K_
+packet|pts=527360|dts=527360|duration=1024|flags=K_
+packet|pts=528384|dts=528384|duration=816|flags=K_
 stream|nb_read_packets=518
 frame|pkt_pts=0|pkt_dts=0|best_effort_timestamp=0|pkt_duration=1024|nb_samples=1024
 frame|pkt_pts=1024|pkt_dts=1024|best_effort_timestamp=1024|pkt_duration=1024|nb_samples=1024
diff --git a/tests/ref/fate/gaplessinfo-itunes1 b/tests/ref/fate/gaplessinfo-itunes1
index 717a2a8..c2fb316 100644
--- a/tests/ref/fate/gaplessinfo-itunes1
+++ b/tests/ref/fate/gaplessinfo-itunes1
@@ -7,22 +7,22 @@  duration_ts=1294336
 start_time=0.047889
 duration=29.350023
 [/FORMAT]
-packet|pts=0|dts=0|duration=1024
-packet|pts=1024|dts=1024|duration=1024
-packet|pts=2048|dts=2048|duration=1024
-packet|pts=3072|dts=3072|duration=1024
-packet|pts=4096|dts=4096|duration=1024
-packet|pts=5120|dts=5120|duration=1024
-packet|pts=6144|dts=6144|duration=1024
-packet|pts=7168|dts=7168|duration=1024
-packet|pts=1286144|dts=1286144|duration=1024
-packet|pts=1287168|dts=1287168|duration=1024
-packet|pts=1288192|dts=1288192|duration=1024
-packet|pts=1289216|dts=1289216|duration=1024
-packet|pts=1290240|dts=1290240|duration=1024
-packet|pts=1291264|dts=1291264|duration=1024
-packet|pts=1292288|dts=1292288|duration=1024
-packet|pts=1293312|dts=1293312|duration=1024
+packet|pts=0|dts=0|duration=1024|flags=K_
+packet|pts=1024|dts=1024|duration=1024|flags=K_
+packet|pts=2048|dts=2048|duration=1024|flags=K_
+packet|pts=3072|dts=3072|duration=1024|flags=K_
+packet|pts=4096|dts=4096|duration=1024|flags=K_
+packet|pts=5120|dts=5120|duration=1024|flags=K_
+packet|pts=6144|dts=6144|duration=1024|flags=K_
+packet|pts=7168|dts=7168|duration=1024|flags=K_
+packet|pts=1286144|dts=1286144|duration=1024|flags=K_
+packet|pts=1287168|dts=1287168|duration=1024|flags=K_
+packet|pts=1288192|dts=1288192|duration=1024|flags=K_
+packet|pts=1289216|dts=1289216|duration=1024|flags=K_
+packet|pts=1290240|dts=1290240|duration=1024|flags=K_
+packet|pts=1291264|dts=1291264|duration=1024|flags=K_
+packet|pts=1292288|dts=1292288|duration=1024|flags=K_
+packet|pts=1293312|dts=1293312|duration=1024|flags=K_
 stream|nb_read_packets=1264
 frame|pkt_pts=2112|pkt_dts=2112|best_effort_timestamp=2048|pkt_duration=960|nb_samples=960
 frame|pkt_pts=3072|pkt_dts=3072|best_effort_timestamp=3072|pkt_duration=1024|nb_samples=1024
diff --git a/tests/ref/fate/gaplessinfo-itunes2 b/tests/ref/fate/gaplessinfo-itunes2
index be15113..073715e 100644
--- a/tests/ref/fate/gaplessinfo-itunes2
+++ b/tests/ref/fate/gaplessinfo-itunes2
@@ -7,22 +7,22 @@  duration_ts=105472
 start_time=0.047891
 duration=2.391655
 [/FORMAT]
-packet|pts=0|dts=0|duration=1024
-packet|pts=1024|dts=1024|duration=1024
-packet|pts=2048|dts=2048|duration=1024
-packet|pts=3072|dts=3072|duration=1024
-packet|pts=4096|dts=4096|duration=1024
-packet|pts=5120|dts=5120|duration=1024
-packet|pts=6144|dts=6144|duration=1024
-packet|pts=7168|dts=7168|duration=1024
-packet|pts=97280|dts=97280|duration=1024
-packet|pts=98304|dts=98304|duration=1024
-packet|pts=99328|dts=99328|duration=1024
-packet|pts=100352|dts=100352|duration=1024
-packet|pts=101376|dts=101376|duration=1024
-packet|pts=102400|dts=102400|duration=1024
-packet|pts=103424|dts=103424|duration=1024
-packet|pts=104448|dts=104448|duration=1024
+packet|pts=0|dts=0|duration=1024|flags=K_
+packet|pts=1024|dts=1024|duration=1024|flags=K_
+packet|pts=2048|dts=2048|duration=1024|flags=K_
+packet|pts=3072|dts=3072|duration=1024|flags=K_
+packet|pts=4096|dts=4096|duration=1024|flags=K_
+packet|pts=5120|dts=5120|duration=1024|flags=K_
+packet|pts=6144|dts=6144|duration=1024|flags=K_
+packet|pts=7168|dts=7168|duration=1024|flags=K_
+packet|pts=97280|dts=97280|duration=1024|flags=K_
+packet|pts=98304|dts=98304|duration=1024|flags=K_
+packet|pts=99328|dts=99328|duration=1024|flags=K_
+packet|pts=100352|dts=100352|duration=1024|flags=K_
+packet|pts=101376|dts=101376|duration=1024|flags=K_
+packet|pts=102400|dts=102400|duration=1024|flags=K_
+packet|pts=103424|dts=103424|duration=1024|flags=K_
+packet|pts=104448|dts=104448|duration=1024|flags=K_
 stream|nb_read_packets=103
 frame|pkt_pts=2112|pkt_dts=2112|best_effort_timestamp=2048|pkt_duration=960|nb_samples=960
 frame|pkt_pts=3072|pkt_dts=3072|best_effort_timestamp=3072|pkt_duration=1024|nb_samples=1024